FAQ
Author: gates
Date: Thu Mar 12 20:01:30 2015
New Revision: 1666285

URL: http://svn.apache.org/r1666285
Log:
HIVE-9874 Partition storage descriptors being set from table sd without copying [hbase-metastore branch] (Alan Gates)

Added:
     hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
Modified:
     hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
     hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
     hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
     hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
     hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
     hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
     hive/branches/hbase-metastore/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Mar 12 20:01:30 2015
@@ -1879,7 +1879,7 @@ public class HiveMetaStore extends Thrif

          firePreEvent(new PreAddPartitionEvent(tbl, part, this));

- part.setSd(tbl.getSd());
+ part.setSd(tbl.getSd().deepCopy());
          partLocation = new Path(tbl.getSd().getLocation(), Warehouse
              .makePartName(tbl.getPartitionKeys(), part_vals));
          part.getSd().setLocation(partLocation.toString());

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java Thu Mar 12 20:01:30 2015
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
  import org.apache.hadoop.hive.metastore.api.Role;
  import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.metastore.api.Table;
  import org.apache.hadoop.hive.metastore.api.Type;
  import org.apache.hadoop.hive.metastore.api.UnknownDBException;

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java Thu Mar 12 20:01:30 2015
@@ -640,12 +640,14 @@ class HBaseUtils {
    }

    static void assembleStorageDescriptor(StorageDescriptor sd, StorageDescriptorParts parts) {
- sd.setLocation(parts.location);
- sd.setParameters(parts.parameters);
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setLocation(parts.location);
+ ssd.setParameters(parts.parameters);
+ ssd.setShared(sd);
      if (parts.containingPartition != null) {
- parts.containingPartition.setSd(sd);
+ parts.containingPartition.setSd(ssd);
      } else if (parts.containingTable != null) {
- parts.containingTable.setSd(sd);
+ parts.containingTable.setSd(ssd);
      } else {
        throw new RuntimeException("Need either a partition or a table");
      }

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java Thu Mar 12 20:01:30 2015
@@ -26,7 +26,8 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.SkewedInfo;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;

-import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
@@ -38,7 +39,11 @@ import java.util.Map;
   */
  public class SharedStorageDescriptor extends StorageDescriptor {
    static final private Log LOG = LogFactory.getLog(SharedStorageDescriptor.class.getName());
- StorageDescriptor shared;
+ private StorageDescriptor shared;
+ private boolean copied = false;
+ private CopyOnWriteColList colList = null;
+ private CopyOnWriteOrderList orderList = null;
+ private CopyOnWriteBucketList bucketList = null;

    SharedStorageDescriptor() {
    }
@@ -49,13 +54,20 @@ public class SharedStorageDescriptor ext
      this.shared = that.shared;
    }

- void readShared(byte[] hash) throws IOException {
- shared = HBaseReadWrite.getInstance().getStorageDescriptor(hash);
+ @Override
+ public StorageDescriptor deepCopy() {
+ return new SharedStorageDescriptor(this);
+ }
+
+ @Override
+ public boolean isSetCols() {
+ return shared.isSetCols();
    }

    @Override
    public List<FieldSchema> getCols() {
- return shared.getCols();
+ return copied ? shared.getCols() : (
+ shared.getCols() == null ? null : copyCols(shared.getCols()));
    }

    @Override
@@ -69,33 +81,143 @@ public class SharedStorageDescriptor ext
    }

    @Override
+ public void setCols(List<FieldSchema> cols) {
+ copyOnWrite();
+ shared.setCols(cols);
+ }
+
+ @Override
+ public void addToCols(FieldSchema fs) {
+ copyOnWrite();
+ shared.addToCols(fs);
+ }
+
+ @Override
+ public void unsetCols() {
+ copyOnWrite();
+ shared.unsetCols();
+ }
+
+ @Override
+ public boolean isSetInputFormat() {
+ return shared.isSetInputFormat();
+ }
+
+ @Override
    public String getInputFormat() {
      return shared.getInputFormat();
    }

    @Override
+ public void setInputFormat(String inputFormat) {
+ copyOnWrite();
+ shared.setInputFormat(inputFormat);
+ }
+
+ @Override
+ public void unsetInputFormat() {
+ copyOnWrite();
+ shared.unsetInputFormat();
+ }
+
+ @Override
+ public boolean isSetOutputFormat() {
+ return shared.isSetOutputFormat();
+ }
+
+ @Override
    public String getOutputFormat() {
      return shared.getOutputFormat();
    }

    @Override
+ public void setOutputFormat(String outputFormat) {
+ copyOnWrite();
+ shared.setOutputFormat(outputFormat);
+ }
+
+ @Override
+ public void unsetOutputFormat() {
+ copyOnWrite();
+ shared.unsetOutputFormat();
+ }
+
+ @Override
+ public boolean isSetCompressed() {
+ return shared.isSetCompressed();
+ }
+
+ @Override
    public boolean isCompressed() {
      return shared.isCompressed();
    }

    @Override
+ public void setCompressed(boolean isCompressed) {
+ copyOnWrite();
+ shared.setCompressed(isCompressed);
+ }
+
+ @Override
+ public void unsetCompressed() {
+ copyOnWrite();
+ shared.unsetCompressed();
+ }
+
+ @Override
+ public boolean isSetNumBuckets() {
+ return shared.isSetNumBuckets();
+ }
+
+ @Override
    public int getNumBuckets() {
      return shared.getNumBuckets();
    }

    @Override
+ public void setNumBuckets(int numBuckets) {
+ copyOnWrite();
+ shared.setNumBuckets(numBuckets);
+ }
+
+ @Override
+ public void unsetNumBuckets() {
+ copyOnWrite();
+ shared.unsetNumBuckets();
+ }
+
+ @Override
+ public boolean isSetSerdeInfo() {
+ return shared.isSetSerdeInfo();
+ }
+
+ @Override
    public SerDeInfo getSerdeInfo() {
- return shared.getSerdeInfo();
+ return copied ? shared.getSerdeInfo() : (
+ shared.getSerdeInfo() == null ? null : new SerDeInfoWrapper(shared.getSerdeInfo()));
+ }
+
+ @Override
+ public void setSerdeInfo(SerDeInfo serdeInfo) {
+ copyOnWrite();
+ shared.setSerdeInfo(serdeInfo);
+ }
+
+ @Override
+ public void unsetSerdeInfo() {
+ copyOnWrite();
+ shared.unsetSerdeInfo();
+ }
+
+ @Override
+ public boolean isSetBucketCols() {
+ return shared.isSetBucketCols();
    }

    @Override
    public List<String> getBucketCols() {
- return shared.getBucketCols();
+ return copied ? shared.getBucketCols() : (
+ shared.getBucketCols() == null ? null : copyBucketCols(shared.getBucketCols()));
    }

    @Override
@@ -109,8 +231,32 @@ public class SharedStorageDescriptor ext
    }

    @Override
+ public void setBucketCols(List<String> bucketCols) {
+ copyOnWrite();
+ shared.setBucketCols(bucketCols);
+ }
+
+ @Override
+ public void addToBucketCols(String bucketCol) {
+ copyOnWrite();
+ shared.addToBucketCols(bucketCol);
+ }
+
+ @Override
+ public void unsetBucketCols() {
+ copyOnWrite();
+ shared.unsetBucketCols();
+ }
+
+ @Override
+ public boolean isSetSortCols() {
+ return shared.isSetSortCols();
+ }
+
+ @Override
    public List<Order> getSortCols() {
- return shared.getSortCols();
+ return copied ? shared.getSortCols() : (
+ shared.getSortCols() == null ? null : copySort(shared.getSortCols()));
    }

    @Override
@@ -124,12 +270,508 @@ public class SharedStorageDescriptor ext
    }

    @Override
+ public void setSortCols(List<Order> sortCols) {
+ copyOnWrite();
+ shared.setSortCols(sortCols);
+ }
+
+ @Override
+ public void addToSortCols(Order sortCol) {
+ copyOnWrite();
+ shared.addToSortCols(sortCol);
+ }
+
+ @Override
+ public void unsetSortCols() {
+ copyOnWrite();
+ shared.unsetSortCols();
+ }
+
+ @Override
+ public boolean isSetSkewedInfo() {
+ return shared.isSetSkewedInfo();
+ }
+
+ @Override
    public SkewedInfo getSkewedInfo() {
- return shared.getSkewedInfo();
+ return copied ? shared.getSkewedInfo() : (
+ shared.getSkewedInfo() == null ? null : new SkewWrapper(shared.getSkewedInfo()));
+ }
+
+ @Override
+ public void setSkewedInfo(SkewedInfo skewedInfo) {
+ copyOnWrite();
+ shared.setSkewedInfo(skewedInfo);
+ }
+
+ @Override
+ public void unsetSkewedInfo() {
+ copyOnWrite();
+ shared.unsetSkewedInfo();
+ }
+
+ @Override
+ public boolean isSetStoredAsSubDirectories() {
+ return shared.isSetStoredAsSubDirectories();
    }

    @Override
    public boolean isStoredAsSubDirectories() {
      return shared.isStoredAsSubDirectories();
    }
+
+ @Override
+ public void setStoredAsSubDirectories(boolean sasd) {
+ copyOnWrite();
+ shared.setStoredAsSubDirectories(sasd);
+ }
+
+ @Override
+ public void unsetStoredAsSubDirectories() {
+ copyOnWrite();
+ shared.unsetStoredAsSubDirectories();
+ }
+
+ void setShared(StorageDescriptor sd) {
+ shared = sd;
+ }
+
+ StorageDescriptor getShared() {
+ return shared;
+ }
+
+ private void copyOnWrite() {
+ if (!copied) {
+ shared = new StorageDescriptor(shared);
+ copied = true;
+ }
+ }
+
+ private class SerDeInfoWrapper extends SerDeInfo {
+
+ SerDeInfoWrapper(SerDeInfo serde) {
+ super(serde);
+ }
+
+ @Override
+ public void setName(String name) {
+ copyOnWrite();
+ shared.getSerdeInfo().setName(name);
+ }
+
+ @Override
+ public void unsetName() {
+ copyOnWrite();
+ shared.getSerdeInfo().unsetName();
+ }
+
+ @Override
+ public void setSerializationLib(String lib) {
+ copyOnWrite();
+ shared.getSerdeInfo().setSerializationLib(lib);
+ }
+
+ @Override
+ public void unsetSerializationLib() {
+ copyOnWrite();
+ shared.getSerdeInfo().unsetSerializationLib();
+ }
+
+ @Override
+ public void setParameters(Map<String, String> parameters) {
+ copyOnWrite();
+ shared.getSerdeInfo().setParameters(parameters);
+ }
+
+ @Override
+ public void unsetParameters() {
+ copyOnWrite();
+ shared.getSerdeInfo().unsetParameters();
+ }
+
+ @Override
+ public void putToParameters(String key, String value) {
+ copyOnWrite();
+ shared.getSerdeInfo().putToParameters(key, value);
+ }
+ }
+
+ private class SkewWrapper extends SkewedInfo {
+ SkewWrapper(SkewedInfo skew) {
+ super(skew);
+ }
+
+ @Override
+ public void setSkewedColNames(List<String> skewedColNames) {
+ copyOnWrite();
+ shared.getSkewedInfo().setSkewedColNames(skewedColNames);
+ }
+
+ @Override
+ public void unsetSkewedColNames() {
+ copyOnWrite();
+ shared.getSkewedInfo().unsetSkewedColNames();
+ }
+
+ @Override
+ public void addToSkewedColNames(String skewCol) {
+ copyOnWrite();
+ shared.getSkewedInfo().addToSkewedColNames(skewCol);
+ }
+
+ @Override
+ public void setSkewedColValues(List<List<String>> skewedColValues) {
+ copyOnWrite();
+ shared.getSkewedInfo().setSkewedColValues(skewedColValues);
+ }
+
+ @Override
+ public void unsetSkewedColValues() {
+ copyOnWrite();
+ shared.getSkewedInfo().unsetSkewedColValues();
+ }
+
+ @Override
+ public void addToSkewedColValues(List<String> skewedColValue) {
+ copyOnWrite();
+ shared.getSkewedInfo().addToSkewedColValues(skewedColValue);
+ }
+
+ @Override
+ public void setSkewedColValueLocationMaps(Map<List<String>, String> maps) {
+ copyOnWrite();
+ shared.getSkewedInfo().setSkewedColValueLocationMaps(maps);
+ }
+
+ @Override
+ public void unsetSkewedColValueLocationMaps() {
+ copyOnWrite();
+ shared.getSkewedInfo().unsetSkewedColValueLocationMaps();
+ }
+
+ @Override
+ public void putToSkewedColValueLocationMaps(List<String> key, String value) {
+ copyOnWrite();
+ shared.getSkewedInfo().putToSkewedColValueLocationMaps(key, value);
+ }
+ }
+
+ private CopyOnWriteOrderList copySort(List<Order> sort) {
+ if (orderList == null) {
+ orderList = new CopyOnWriteOrderList(sort.size());
+ for (int i = 0; i < sort.size(); i++) {
+ orderList.secretAdd(new OrderWrapper(i, sort.get(i)));
+ }
+ }
+ return orderList;
+ }
+
+ private class CopyOnWriteOrderList extends ArrayList<Order> {
+
+ CopyOnWriteOrderList(int size) {
+ super(size);
+ }
+
+ private void secretAdd(OrderWrapper order) {
+ super.add(order);
+ }
+
+ @Override
+ public boolean add(Order t) {
+ copyOnWrite();
+ return shared.getSortCols().add(t);
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ copyOnWrite();
+ return shared.getSortCols().remove(o);
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends Order> c) {
+ copyOnWrite();
+ return shared.getSortCols().addAll(c);
+ }
+
+ @Override
+ public boolean addAll(int index, Collection<? extends Order> c) {
+ copyOnWrite();
+ return shared.getSortCols().addAll(c);
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ copyOnWrite();
+ return shared.getSortCols().removeAll(c);
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ copyOnWrite();
+ return shared.getSortCols().retainAll(c);
+ }
+
+ @Override
+ public void clear() {
+ copyOnWrite();
+ shared.getSortCols().clear();
+ }
+
+ @Override
+ public Order set(int index, Order element) {
+ copyOnWrite();
+ return shared.getSortCols().set(index, element);
+ }
+
+ @Override
+ public void add(int index, Order element) {
+ copyOnWrite();
+ shared.getSortCols().add(index, element);
+ }
+
+ @Override
+ public Order remove(int index) {
+ copyOnWrite();
+ return shared.getSortCols().remove(index);
+ }
+ }
+
+ private class OrderWrapper extends Order {
+ final private int pos;
+
+ OrderWrapper(int pos, Order order) {
+ super(order);
+ this.pos = pos;
+ }
+
+ @Override
+ public void setCol(String col) {
+ copyOnWrite();
+ shared.getSortCols().get(pos).setCol(col);
+ }
+
+ @Override
+ public void unsetCol() {
+ copyOnWrite();
+ shared.getSortCols().get(pos).unsetCol();
+ }
+
+ @Override
+ public void setOrder(int order) {
+ copyOnWrite();
+ shared.getSortCols().get(pos).setOrder(order);
+ }
+
+ @Override
+ public void unsetOrder() {
+ copyOnWrite();
+ shared.getSortCols().get(pos).unsetOrder();
+ }
+ }
+
+ private CopyOnWriteColList copyCols(List<FieldSchema> cols) {
+ if (colList == null) {
+ colList = new CopyOnWriteColList(cols.size());
+ for (int i = 0; i < cols.size(); i++) {
+ colList.secretAdd(new FieldSchemaWrapper(i, cols.get(i)));
+ }
+ }
+ return colList;
+ }
+
+ private class CopyOnWriteColList extends ArrayList<FieldSchema> {
+
+ CopyOnWriteColList(int size) {
+ super(size);
+ }
+
+ private void secretAdd(FieldSchemaWrapper col) {
+ super.add(col);
+ }
+
+ @Override
+ public boolean add(FieldSchema t) {
+ copyOnWrite();
+ return shared.getCols().add(t);
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ copyOnWrite();
+ return shared.getCols().remove(o);
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends FieldSchema> c) {
+ copyOnWrite();
+ return shared.getCols().addAll(c);
+ }
+
+ @Override
+ public boolean addAll(int index, Collection<? extends FieldSchema> c) {
+ copyOnWrite();
+ return shared.getCols().addAll(c);
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ copyOnWrite();
+ return shared.getCols().removeAll(c);
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ copyOnWrite();
+ return shared.getCols().retainAll(c);
+ }
+
+ @Override
+ public void clear() {
+ copyOnWrite();
+ shared.getCols().clear();
+ }
+
+ @Override
+ public FieldSchema set(int index, FieldSchema element) {
+ copyOnWrite();
+ return shared.getCols().set(index, element);
+ }
+
+ @Override
+ public void add(int index, FieldSchema element) {
+ copyOnWrite();
+ shared.getCols().add(index, element);
+ }
+
+ @Override
+ public FieldSchema remove(int index) {
+ copyOnWrite();
+ return shared.getCols().remove(index);
+ }
+ }
+
+ private class FieldSchemaWrapper extends FieldSchema {
+ final private int pos;
+
+ FieldSchemaWrapper(int pos, FieldSchema col) {
+ super(col);
+ this.pos = pos;
+ }
+
+ @Override
+ public void setName(String name) {
+ copyOnWrite();
+ shared.getCols().get(pos).setName(name);
+ }
+
+ @Override
+ public void unsetName() {
+ copyOnWrite();
+ shared.getCols().get(pos).unsetName();
+ }
+
+ @Override
+ public void setType(String type) {
+ copyOnWrite();
+ shared.getCols().get(pos).setType(type);
+ }
+
+ @Override
+ public void unsetType() {
+ copyOnWrite();
+ shared.getCols().get(pos).unsetType();
+ }
+
+ @Override
+ public void setComment(String comment) {
+ copyOnWrite();
+ shared.getCols().get(pos).setComment(comment);
+ }
+
+ @Override
+ public void unsetComment() {
+ copyOnWrite();
+ shared.getCols().get(pos).unsetComment();
+ }
+ }
+
+ private CopyOnWriteBucketList copyBucketCols(List<String> cols) {
+ if (bucketList == null) {
+ bucketList = new CopyOnWriteBucketList(cols);
+ }
+ return bucketList;
+ }
+
+ private class CopyOnWriteBucketList extends ArrayList<String> {
+
+ CopyOnWriteBucketList(Collection<String> c) {
+ super(c);
+ }
+
+ private void secretAdd(String col) {
+ super.add(col);
+ }
+
+ @Override
+ public boolean add(String t) {
+ copyOnWrite();
+ return shared.getBucketCols().add(t);
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ copyOnWrite();
+ return shared.getBucketCols().remove(o);
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends String> c) {
+ copyOnWrite();
+ return shared.getBucketCols().addAll(c);
+ }
+
+ @Override
+ public boolean addAll(int index, Collection<? extends String> c) {
+ copyOnWrite();
+ return shared.getBucketCols().addAll(c);
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ copyOnWrite();
+ return shared.getBucketCols().removeAll(c);
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ copyOnWrite();
+ return shared.getBucketCols().retainAll(c);
+ }
+
+ @Override
+ public void clear() {
+ copyOnWrite();
+ shared.getBucketCols().clear();
+ }
+
+ @Override
+ public String set(int index, String element) {
+ copyOnWrite();
+ return shared.getBucketCols().set(index, element);
+ }
+
+ @Override
+ public void add(int index, String element) {
+ copyOnWrite();
+ shared.getBucketCols().add(index, element);
+ }
+
+ @Override
+ public String remove(int index) {
+ copyOnWrite();
+ return shared.getBucketCols().remove(index);
+ }
+ }
+
  }

Added: hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java?rev=1666285&view=auto
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java (added)
+++ hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java Thu Mar 12 20:01:30 2015
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.hbase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+
+/**
+ *
+ */
+public class TestSharedStorageDescriptor {
+ private static final Log LOG = LogFactory.getLog(TestHBaseStore.class.getName());
+
+
+ @Test
+ public void location() {
+ StorageDescriptor sd = new StorageDescriptor();
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setLocation("here");
+ ssd.setShared(sd);
+ ssd.setLocation("there");
+ Assert.assertTrue(sd == ssd.getShared());
+ }
+
+ @Test
+ public void changeOnInputFormat() {
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setInputFormat("input");
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setShared(sd);
+ Assert.assertEquals("input", ssd.getInputFormat());
+ ssd.setInputFormat("different");
+ Assert.assertFalse(sd == ssd.getShared());
+ Assert.assertEquals("input", sd.getInputFormat());
+ Assert.assertEquals("different", ssd.getInputFormat());
+ Assert.assertEquals("input", sd.getInputFormat());
+ }
+
+ @Test
+ public void changeOnSerde() {
+ StorageDescriptor sd = new StorageDescriptor();
+ SerDeInfo serde = new SerDeInfo();
+ serde.setName("serde");
+ sd.setSerdeInfo(serde);
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setShared(sd);
+ Assert.assertEquals("serde", ssd.getSerdeInfo().getName());
+ ssd.getSerdeInfo().setName("different");
+ Assert.assertFalse(sd == ssd.getShared());
+ Assert.assertEquals("serde", serde.getName());
+ Assert.assertEquals("different", ssd.getSerdeInfo().getName());
+ Assert.assertEquals("serde", sd.getSerdeInfo().getName());
+ }
+
+ @Test
+ public void multipleChangesDontCauseMultipleCopies() {
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.setInputFormat("input");
+ sd.setOutputFormat("output");
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setShared(sd);
+ Assert.assertEquals("input", ssd.getInputFormat());
+ ssd.setInputFormat("different");
+ Assert.assertFalse(sd == ssd.getShared());
+ Assert.assertEquals("input", sd.getInputFormat());
+ Assert.assertEquals("different", ssd.getInputFormat());
+ StorageDescriptor keep = ssd.getShared();
+ ssd.setOutputFormat("different_output");
+ Assert.assertEquals("different", ssd.getInputFormat());
+ Assert.assertEquals("different_output", ssd.getOutputFormat());
+ Assert.assertEquals("output", sd.getOutputFormat());
+ Assert.assertTrue(keep == ssd.getShared());
+ }
+
+ @Test
+ public void changeOrder() {
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.addToSortCols(new Order("fred", 1));
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setShared(sd);
+ Assert.assertEquals(1, ssd.getSortCols().get(0).getOrder());
+ ssd.getSortCols().get(0).setOrder(2);
+ Assert.assertFalse(sd == ssd.getShared());
+ Assert.assertEquals(2, ssd.getSortCols().get(0).getOrder());
+ Assert.assertEquals(1, sd.getSortCols().get(0).getOrder());
+ }
+
+ @Test
+ public void changeOrderList() {
+ StorageDescriptor sd = new StorageDescriptor();
+ sd.addToSortCols(new Order("fred", 1));
+ SharedStorageDescriptor ssd = new SharedStorageDescriptor();
+ ssd.setShared(sd);
+ Assert.assertEquals(1, ssd.getSortCols().get(0).getOrder());
+ List<Order> list = ssd.getSortCols();
+ list.add(new Order("bob", 2));
+ Assert.assertFalse(sd == ssd.getShared());
+ Assert.assertEquals(2, ssd.getSortColsSize());
+ Assert.assertEquals(1, sd.getSortColsSize());
+ }
+
+}

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Thu Mar 12 20:01:30 2015
@@ -39,17 +39,12 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.FieldSchema;
  import org.apache.hadoop.hive.metastore.api.MetaException;
  import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.hbase.SharedStorageDescriptor;
  import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
  import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
  import org.apache.hadoop.hive.serde2.Deserializer;
  import org.apache.hadoop.mapred.InputFormat;
  import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.transport.TMemoryBuffer;

  /**
   * A Hive Table Partition: is a fundamental storage unit within a Table.
@@ -96,7 +91,7 @@ public class Partition implements Serial
      org.apache.hadoop.hive.metastore.api.Partition tPart =
          new org.apache.hadoop.hive.metastore.api.Partition();
      if (!tbl.isView()) {
- tPart.setSd(tbl.getTTable().getSd()); // TODO: get a copy
+ tPart.setSd(tbl.getTTable().getSd().deepCopy());
      }
      initialize(tbl, tPart);
    }
@@ -141,37 +136,13 @@ public class Partition implements Serial
      tpart.setValues(pvals);

      if (!tbl.isView()) {
- tpart.setSd(cloneSd(tbl));
+ tpart.setSd(tbl.getSd().deepCopy());
        tpart.getSd().setLocation((location != null) ? location.toString() : null);
      }
      return tpart;
    }

    /**
- * We already have methods that clone stuff using XML or Kryo.
- * And now for something completely different - let's clone SD using Thrift!
- * Refactored into a method.
- */
- public static StorageDescriptor cloneSd(Table tbl) throws HiveException {
- if (tbl.getSd() instanceof SharedStorageDescriptor) {
- return new SharedStorageDescriptor((SharedStorageDescriptor)tbl.getSd());
- }
- // What is the point of this? Why not just use the copy constructor in StorageDescriptor?
- StorageDescriptor sd = new StorageDescriptor();
- try {
- // replace with THRIFT-138
- TMemoryBuffer buffer = new TMemoryBuffer(1024);
- TBinaryProtocol prot = new TBinaryProtocol(buffer);
- tbl.getTTable().getSd().write(prot);
- sd.read(prot);
- } catch (TException e) {
- LOG.error("Could not create a copy of StorageDescription");
- throw new HiveException("Could not create a copy of StorageDescription",e);
- }
- return sd;
- }
-
- /**
     * Initializes this object with the given variables
     *
     * @param table

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java Thu Mar 12 20:01:30 2015
@@ -415,7 +415,7 @@ public class AuthorizationPreEventListen
          // location or an SD, but these are needed to create a ql.metadata.Partition,
          // so we use the table's SD. The only place this is used is by the
          // authorization hooks, so we will not affect code flow in the metastore itself.
- wrapperApiPart.setSd(t.getSd());
+ wrapperApiPart.setSd(t.getSd().deepCopy());
        }
        initialize(new TableWrapper(t),wrapperApiPart);
      }

Modified: hive/branches/hbase-metastore/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java?rev=1666285&r1=1666284&r2=1666285&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java (original)
+++ hive/branches/hbase-metastore/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java Thu Mar 12 20:01:30 2015
@@ -257,7 +257,7 @@ public class TestMetastoreExpr extends T
      part.setTableName(table.getTableName());
      part.setValues(vals);
      part.setParameters(new HashMap<String, String>());
- part.setSd(table.getSd());
+ part.setSd(table.getSd().deepCopy());
      part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
      part.getSd().setLocation(table.getSd().getLocation() + location);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 12, '15 at 8:01p
activeMar 12, '15 at 8:01p
posts1
users1
websitehive.apache.org

1 user in discussion

Gates: 1 post

People

Translate

site design / logo © 2021 Grokbase