FAQ
Modified: hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Fri Apr 12 22:10:31 2013
@@ -1232,6 +1232,74 @@ class Order:
    def __ne__(self, other):
      return not (self == other)

+class SkewedValueList:
+ """
+ Attributes:
+ - skewedValueList
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.LIST, 'skewedValueList', (TType.STRING,None), None, ), # 1
+ )
+
+ def __init__(self, skewedValueList=None,):
+ self.skewedValueList = skewedValueList
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.LIST:
+ self.skewedValueList = []
+ (_etype90, _size87) = iprot.readListBegin()
+ for _i91 in xrange(_size87):
+ _elem92 = iprot.readString();
+ self.skewedValueList.append(_elem92)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SkewedValueList')
+ if self.skewedValueList is not None:
+ oprot.writeFieldBegin('skewedValueList', TType.LIST, 1)
+ oprot.writeListBegin(TType.STRING, len(self.skewedValueList))
+ for iter93 in self.skewedValueList:
+ oprot.writeString(iter93)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
  class SkewedInfo:
    """
    Attributes:
@@ -1244,7 +1312,7 @@ class SkewedInfo:
      None, # 0
      (1, TType.LIST, 'skewedColNames', (TType.STRING,None), None, ), # 1
      (2, TType.LIST, 'skewedColValues', (TType.LIST,(TType.STRING,None)), None, ), # 2
- (3, TType.MAP, 'skewedColValueLocationMaps', (TType.LIST,(TType.STRING,None),TType.STRING,None), None, ), # 3
+ (3, TType.MAP, 'skewedColValueLocationMaps', (TType.STRUCT,(SkewedValueList, SkewedValueList.thrift_spec),TType.STRING,None), None, ), # 3
    )

    def __init__(self, skewedColNames=None, skewedColValues=None, skewedColValueLocationMaps=None,):
@@ -1264,41 +1332,37 @@ class SkewedInfo:
        if fid == 1:
          if ftype == TType.LIST:
            self.skewedColNames = []
- (_etype90, _size87) = iprot.readListBegin()
- for _i91 in xrange(_size87):
- _elem92 = iprot.readString();
- self.skewedColNames.append(_elem92)
+ (_etype97, _size94) = iprot.readListBegin()
+ for _i98 in xrange(_size94):
+ _elem99 = iprot.readString();
+ self.skewedColNames.append(_elem99)
            iprot.readListEnd()
          else:
            iprot.skip(ftype)
        elif fid == 2:
          if ftype == TType.LIST:
            self.skewedColValues = []
- (_etype96, _size93) = iprot.readListBegin()
- for _i97 in xrange(_size93):
- _elem98 = []
- (_etype102, _size99) = iprot.readListBegin()
- for _i103 in xrange(_size99):
- _elem104 = iprot.readString();
- _elem98.append(_elem104)
+ (_etype103, _size100) = iprot.readListBegin()
+ for _i104 in xrange(_size100):
+ _elem105 = []
+ (_etype109, _size106) = iprot.readListBegin()
+ for _i110 in xrange(_size106):
+ _elem111 = iprot.readString();
+ _elem105.append(_elem111)
              iprot.readListEnd()
- self.skewedColValues.append(_elem98)
+ self.skewedColValues.append(_elem105)
            iprot.readListEnd()
          else:
            iprot.skip(ftype)
        elif fid == 3:
          if ftype == TType.MAP:
            self.skewedColValueLocationMaps = {}
- (_ktype106, _vtype107, _size105 ) = iprot.readMapBegin()
- for _i109 in xrange(_size105):
- _key110 = []
- (_etype115, _size112) = iprot.readListBegin()
- for _i116 in xrange(_size112):
- _elem117 = iprot.readString();
- _key110.append(_elem117)
- iprot.readListEnd()
- _val111 = iprot.readString();
- self.skewedColValueLocationMaps[_key110] = _val111
+ (_ktype113, _vtype114, _size112 ) = iprot.readMapBegin()
+ for _i116 in xrange(_size112):
+ _key117 = SkewedValueList()
+ _key117.read(iprot)
+ _val118 = iprot.readString();
+ self.skewedColValueLocationMaps[_key117] = _val118
            iprot.readMapEnd()
          else:
            iprot.skip(ftype)
@@ -1315,29 +1379,26 @@ class SkewedInfo:
      if self.skewedColNames is not None:
        oprot.writeFieldBegin('skewedColNames', TType.LIST, 1)
        oprot.writeListBegin(TType.STRING, len(self.skewedColNames))
- for iter118 in self.skewedColNames:
- oprot.writeString(iter118)
+ for iter119 in self.skewedColNames:
+ oprot.writeString(iter119)
        oprot.writeListEnd()
        oprot.writeFieldEnd()
      if self.skewedColValues is not None:
        oprot.writeFieldBegin('skewedColValues', TType.LIST, 2)
        oprot.writeListBegin(TType.LIST, len(self.skewedColValues))
- for iter119 in self.skewedColValues:
- oprot.writeListBegin(TType.STRING, len(iter119))
- for iter120 in iter119:
- oprot.writeString(iter120)
+ for iter120 in self.skewedColValues:
+ oprot.writeListBegin(TType.STRING, len(iter120))
+ for iter121 in iter120:
+ oprot.writeString(iter121)
          oprot.writeListEnd()
        oprot.writeListEnd()
        oprot.writeFieldEnd()
      if self.skewedColValueLocationMaps is not None:
        oprot.writeFieldBegin('skewedColValueLocationMaps', TType.MAP, 3)
- oprot.writeMapBegin(TType.LIST, TType.STRING, len(self.skewedColValueLocationMaps))
- for kiter121,viter122 in self.skewedColValueLocationMaps.items():
- oprot.writeListBegin(TType.STRING, len(kiter121))
- for iter123 in kiter121:
- oprot.writeString(iter123)
- oprot.writeListEnd()
- oprot.writeString(viter122)
+ oprot.writeMapBegin(TType.STRUCT, TType.STRING, len(self.skewedColValueLocationMaps))
+ for kiter122,viter123 in self.skewedColValueLocationMaps.items():
+ kiter122.write(oprot)
+ oprot.writeString(viter123)
        oprot.writeMapEnd()
        oprot.writeFieldEnd()
      oprot.writeFieldStop()

Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Fri Apr 12 22:10:31 2013
@@ -289,6 +289,22 @@ class Order
    ::Thrift::Struct.generate_accessors self
  end

+class SkewedValueList
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SKEWEDVALUELIST = 1
+
+ FIELDS = {
+ SKEWEDVALUELIST => {:type => ::Thrift::Types::LIST, :name => 'skewedValueList', :element => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
  class SkewedInfo
    include ::Thrift::Struct, ::Thrift::Struct_Union
    SKEWEDCOLNAMES = 1
@@ -298,7 +314,7 @@ class SkewedInfo
    FIELDS = {
      SKEWEDCOLNAMES => {:type => ::Thrift::Types::LIST, :name => 'skewedColNames', :element => {:type => ::Thrift::Types::STRING}},
      SKEWEDCOLVALUES => {:type => ::Thrift::Types::LIST, :name => 'skewedColValues', :element => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}},
- SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}, :value => {:type => ::Thrift::Types::STRING}}
+ SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::STRUCT, :class => ::SkewedValueList}, :value => {:type => ::Thrift::Types::STRING}}
    }

    def struct_fields; FIELDS; end

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Apr 12 22:10:31 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.Role;
  import org.apache.hadoop.hive.metastore.api.SerDeInfo;
  import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
  import org.apache.hadoop.hive.metastore.api.Table;
@@ -1054,13 +1055,13 @@ public class ObjectStore implements RawS
     * @param mMap
     * @return
     */
- private Map<List<String>, String> covertToSkewedMap(Map<MStringList, String> mMap) {
- Map<List<String>, String> map = null;
+ private Map<SkewedValueList, String> covertToSkewedMap(Map<MStringList, String> mMap) {
+ Map<SkewedValueList, String> map = null;
      if (mMap != null) {
- map = new HashMap<List<String>, String>(mMap.size());
+ map = new HashMap<SkewedValueList, String>(mMap.size());
        Set<MStringList> keys = mMap.keySet();
        for (MStringList key : keys) {
- map.put(new ArrayList<String>(key.getInternalList()), mMap.get(key));
+ map.put(new SkewedValueList(new ArrayList<String>(key.getInternalList())), mMap.get(key));
        }
      }
      return map;
@@ -1071,13 +1072,12 @@ public class ObjectStore implements RawS
     * @param mMap
     * @return
     */
- private Map<MStringList, String> covertToMapMStringList(Map<List<String>, String> mMap) {
+ private Map<MStringList, String> convertToMapMStringList(Map<SkewedValueList, String> mMap) {
      Map<MStringList, String> map = null;
      if (mMap != null) {
        map = new HashMap<MStringList, String>(mMap.size());
- Set<List<String>> keys = mMap.keySet();
- for (List<String> key : keys) {
- map.put(new MStringList(key), mMap.get(key));
+ for (Map.Entry<SkewedValueList, String> entry : mMap.entrySet()) {
+ map.put(new MStringList(entry.getKey().getSkewedValueList()), entry.getValue());
        }
      }
      return map;
@@ -1124,7 +1124,7 @@ public class ObjectStore implements RawS
              : sd.getSkewedInfo().getSkewedColNames(),
          convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
              .getSkewedColValues()),
- covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
+ convertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo()
              .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories());
    }


Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Fri Apr 12 22:10:31 2013
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.Partition;
  import org.apache.hadoop.hive.metastore.api.SerDeInfo;
  import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
  import org.apache.hadoop.hive.metastore.api.Table;
@@ -185,8 +186,8 @@ public abstract class TestHiveMetaStore
        skewInfor.setSkewedColNames(Arrays.asList("name"));
        List<String> skv = Arrays.asList("1");
        skewInfor.setSkewedColValues(Arrays.asList(skv));
- Map<List<String>, String> scvlm = new HashMap<List<String>, String>();
- scvlm.put(skv, "location1");
+ Map<SkewedValueList, String> scvlm = new HashMap<SkewedValueList, String>();
+ scvlm.put(new SkewedValueList(skv), "location1");
        skewInfor.setSkewedColValueLocationMaps(scvlm);
        sd.setSkewedInfo(skewInfor);


Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Fri Apr 12 22:10:31 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.ql.ErrorMsg;
  import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
  import org.apache.hadoop.hive.ql.io.HiveKey;
@@ -708,7 +709,7 @@ public class FileSinkOperator extends Te
      List<String> skewedCols = lbCtx.getSkewedColNames();
      List<List<String>> allSkewedVals = lbCtx.getSkewedColValues();
      List<String> skewedValsCandidate = null;
- Map<List<String>, String> locationMap = lbCtx.getLbLocationMap();
+ Map<SkewedValueList, String> locationMap = lbCtx.getLbLocationMap();

      /* Convert input row to standard objects. */
      ObjectInspectorUtils.copyToStandardObject(standObjs, row,
@@ -726,14 +727,14 @@ public class FileSinkOperator extends Te
      if (allSkewedVals.contains(skewedValsCandidate)) {
        /* matches skewed values. */
        lbDirName = FileUtils.makeListBucketingDirName(skewedCols, skewedValsCandidate);
- locationMap.put(skewedValsCandidate, lbDirName);
+ locationMap.put(new SkewedValueList(skewedValsCandidate), lbDirName);
      } else {
        /* create default directory. */
        lbDirName = FileUtils.makeDefaultListBucketingDirName(skewedCols,
            lbCtx.getDefaultDirName());
        List<String> defaultKey = Arrays.asList(lbCtx.getDefaultKey());
        if (!locationMap.containsKey(defaultKey)) {
- locationMap.put(defaultKey, lbDirName);
+ locationMap.put(new SkewedValueList(defaultKey), lbDirName);
        }
      }
      return lbDirName;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Apr 12 22:10:31 2013
@@ -76,6 +76,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.Role;
  import org.apache.hadoop.hive.metastore.api.SerDeInfo;
  import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
  import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
@@ -1220,8 +1221,8 @@ public class Hive {
            org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
            SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
            /* Construct list bucketing location mappings from sub-directory name. */
- Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
- newPartPath, skewedInfo);
+ Map<SkewedValueList, String> skewedColValueLocationMaps =
+ constructListBucketingLocationMap(newPartPath, skewedInfo);
            /* Add list bucketing location mappings. */
            skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
            newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
@@ -1254,7 +1255,8 @@ public class Hive {
   * @throws IOException
   */
  private void walkDirTree(FileStatus fSta, FileSystem fSys,
- Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
+ Map<SkewedValueList, String> skewedColValueLocationMaps,
+ Path newPartPath, SkewedInfo skewedInfo)
      throws IOException {
    /* Base Case. It's leaf. */
    if (!fSta.isDir()) {
@@ -1280,7 +1282,7 @@ private void walkDirTree(FileStatus fSta
   * @param skewedInfo
   */
  private void constructOneLBLocationMap(FileStatus fSta,
- Map<List<String>, String> skewedColValueLocationMaps,
+ Map<SkewedValueList, String> skewedColValueLocationMaps,
      Path newPartPath, SkewedInfo skewedInfo) {
    Path lbdPath = fSta.getPath().getParent();
    List<String> skewedValue = new ArrayList<String>();
@@ -1303,7 +1305,7 @@ private void constructOneLBLocationMap(F
    }
    if ((skewedValue.size() > 0) && (skewedValue.size() == skewedInfo.getSkewedColNames().size())
        && !skewedColValueLocationMaps.containsKey(skewedValue)) {
- skewedColValueLocationMaps.put(skewedValue, lbdPath.toString());
+ skewedColValueLocationMaps.put(new SkewedValueList(skewedValue), lbdPath.toString());
    }
  }

@@ -1316,9 +1318,10 @@ private void constructOneLBLocationMap(F
     * @throws IOException
     * @throws FileNotFoundException
     */
- private Map<List<String>, String> constructListBucketingLocationMap(Path newPartPath,
+ private Map<SkewedValueList, String> constructListBucketingLocationMap(Path newPartPath,
        SkewedInfo skewedInfo) throws IOException, FileNotFoundException {
- Map<List<String>, String> skewedColValueLocationMaps = new HashMap<List<String>, String>();
+ Map<SkewedValueList, String> skewedColValueLocationMaps =
+ new HashMap<SkewedValueList, String>();
      FileSystem fSys = newPartPath.getFileSystem(conf);
      walkDirTree(fSys.getFileStatus(newPartPath), fSys, skewedColValueLocationMaps, newPartPath,
          skewedInfo);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Fri Apr 12 22:10:31 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.FieldSchema;
  import org.apache.hadoop.hive.metastore.api.MetaException;
  import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -647,18 +648,18 @@ public class Partition implements Serial

    public void setSkewedValueLocationMap(List<String> valList, String dirName)
        throws HiveException {
- Map<List<String>, String> mappings = tPartition.getSd().getSkewedInfo()
+ Map<SkewedValueList, String> mappings = tPartition.getSd().getSkewedInfo()
          .getSkewedColValueLocationMaps();
      if (null == mappings) {
- mappings = new HashMap<List<String>, String>();
+ mappings = new HashMap<SkewedValueList, String>();
        tPartition.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
      }

      // Add or update new mapping
- mappings.put(valList, dirName);
+ mappings.put(new SkewedValueList(valList), dirName);
    }

- public Map<List<String>, String> getSkewedColValueLocationMaps() {
+ public Map<SkewedValueList, String> getSkewedColValueLocationMaps() {
      return tPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps();
    }
  }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Fri Apr 12 22:10:31 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.Order;
  import org.apache.hadoop.hive.metastore.api.SerDeInfo;
  import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
  import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -144,7 +145,7 @@ public class Table implements Serializab
        SkewedInfo skewInfo = new SkewedInfo();
        skewInfo.setSkewedColNames(new ArrayList<String>());
        skewInfo.setSkewedColValues(new ArrayList<List<String>>());
- skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
+ skewInfo.setSkewedColValueLocationMaps(new HashMap<SkewedValueList, String>());
        sd.setSkewedInfo(skewInfo);
      }

@@ -518,20 +519,20 @@ public class Table implements Serializab

    public void setSkewedValueLocationMap(List<String> valList, String dirName)
        throws HiveException {
- Map<List<String>, String> mappings = tTable.getSd().getSkewedInfo()
+ Map<SkewedValueList, String> mappings = tTable.getSd().getSkewedInfo()
          .getSkewedColValueLocationMaps();
      if (null == mappings) {
- mappings = new HashMap<List<String>, String>();
+ mappings = new HashMap<SkewedValueList, String>();
        tTable.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
      }

      // Add or update new mapping
- mappings.put(valList, dirName);
+ mappings.put(new SkewedValueList(valList), dirName);
    }

- public Map<List<String>,String> getSkewedColValueLocationMaps() {
+ public Map<SkewedValueList,String> getSkewedColValueLocationMaps() {
      return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo()
- .getSkewedColValueLocationMaps() : new HashMap<List<String>, String>();
+ .getSkewedColValueLocationMaps() : new HashMap<SkewedValueList, String>();
    }

    public void setSkewedColValues(List<List<String>> skewedValues) throws HiveException {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Fri Apr 12 22:10:31 2013
@@ -31,6 +31,7 @@ import org.apache.commons.lang.StringEsc
  import org.apache.hadoop.hive.metastore.TableType;
  import org.apache.hadoop.hive.metastore.api.FieldSchema;
  import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.ql.index.HiveIndex;
  import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType;
@@ -193,7 +194,7 @@ public final class MetaDataFormatUtils {
          formatOutput("Skewed Values:", skewedColValues.toString(), tableInfo);
        }

- Map<List<String>, String> skewedColMap = storageDesc.getSkewedInfo()
+ Map<SkewedValueList, String> skewedColMap = storageDesc.getSkewedInfo()
            .getSkewedColValueLocationMaps();
        if ((skewedColMap!=null) && (skewedColMap.size() > 0)) {
          formatOutput("Skewed Value to Path:", skewedColMap.toString(),
@@ -201,9 +202,8 @@ public final class MetaDataFormatUtils {
          Map<List<String>, String> truncatedSkewedColMap = new HashMap<List<String>, String>();
          // walk through existing map to truncate path so that test won't mask it
          // then we can verify location is right
- Set<Entry<List<String>, String>> entries = skewedColMap.entrySet();
- for (Entry<List<String>, String> entry : entries) {
- truncatedSkewedColMap.put(entry.getKey(),
+ for (Entry<SkewedValueList, String> entry : skewedColMap.entrySet()) {
+ truncatedSkewedColMap.put(entry.getKey().getSkewedValueList(),
                PlanUtils.removePrefixFromWarehouseConfig(entry.getValue()));
          }
          formatOutput("Skewed Value to Truncated Path:",

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java Fri Apr 12 22:10:31 2013
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
  import org.apache.hadoop.hive.ql.metadata.Partition;
  import org.apache.hadoop.hive.ql.optimizer.PrunerUtils;
@@ -290,12 +291,13 @@ public class ListBucketingPruner impleme
        List<List<String>> uniqSkewedValues) throws SemanticException {
      // For each entry in dynamic-multi-dimension collection.
      List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.
- Map<List<String>, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
+ Map<SkewedValueList, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
                                                                                 // map.
      assert ListBucketingPrunerUtils.isListBucketingPart(part) : part.getName()
          + " skewed metadata is corrupted. No skewed column and/or location mappings information.";
      List<List<String>> skewedValues = part.getSkewedColValues();
      List<Boolean> nonSkewedValueMatchResult = new ArrayList<Boolean>();
+ SkewedValueList skewedValueList = new SkewedValueList();
      for (List<String> cell : collections) {
        // Walk through the tree to decide value.
        // Example: skewed column: C1, C2 ;
@@ -309,8 +311,9 @@ public class ListBucketingPruner impleme
            /* It's valid case if a partition: */
            /* 1. is defined with skewed columns and skewed values in metadata */
            /* 2. doesn't have all skewed values within its data */
- if (mappings.get(cell) != null) {
- selectedPaths.add(new Path(mappings.get(cell)));
+ skewedValueList.setSkewedValueList(cell);
+ if (mappings.get(skewedValueList) != null) {
+ selectedPaths.add(new Path(mappings.get(skewedValueList)));
            }
          }
        } else {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Fri Apr 12 22:10:31 2013
@@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFac
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.metastore.api.FieldSchema;
  import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.ql.Context;
  import org.apache.hadoop.hive.ql.ErrorMsg;
  import org.apache.hadoop.hive.ql.QueryProperties;
@@ -965,7 +966,7 @@ public abstract class BaseSemanticAnalyz
     * @return
     */
    protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames,
- List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps,
+ List<List<String>> skewedValues, Map<SkewedValueList, String> skewedColValueLocationMaps,
        boolean isStoredAsSubDirectories, HiveConf conf) {
      ListBucketingCtx lbCtx = new ListBucketingCtx();
      lbCtx.setSkewedColNames(skewedColNames);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java?rev=1467491&r1=1467490&r2=1467491&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java Fri Apr 12 22:10:31 2013
@@ -23,6 +23,7 @@ import java.util.List;
  import java.util.Map;

  import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.SkewedValueList;
  import org.apache.hadoop.hive.ql.exec.ColumnInfo;
  import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
  import org.apache.hadoop.hive.ql.exec.RowSchema;
@@ -39,7 +40,7 @@ public class ListBucketingCtx implements
    private static final long serialVersionUID = 1L;
    private List<String> skewedColNames;
    private List<List<String>> skewedColValues;
- private Map<List<String>, String> lbLocationMap;
+ private Map<SkewedValueList, String> lbLocationMap;
    private List<SkewedColumnPositionPair> rowSkewedIndex;
    private boolean isStoredAsSubDirectories;
    private String defaultKey;
@@ -82,14 +83,14 @@ public class ListBucketingCtx implements
    /**
     * @return the lbLocationMap
     */
- public Map<List<String>, String> getLbLocationMap() {
+ public Map<SkewedValueList, String> getLbLocationMap() {
      return lbLocationMap;
    }

    /**
     * @param lbLocationMap the lbLocationMap to set
     */
- public void setLbLocationMap(Map<List<String>, String> lbLocationMap) {
+ public void setLbLocationMap(Map<SkewedValueList, String> lbLocationMap) {
      this.lbLocationMap = lbLocationMap;
    }

Search Discussions

Discussion Posts

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 12, '13 at 10:10p
activeApr 12, '13 at 10:11p
posts2
users1
websitehive.apache.org

1 user in discussion

Gangtimliu: 2 posts

People

Translate

site design / logo © 2021 Grokbase