FAQ
Author: khorgath
Date: Wed Nov 5 07:43:40 2014
New Revision: 1636814

URL: http://svn.apache.org/r1636814
Log:
HIVE-8719 : LoadSemanticAnalyzer ignores previous partition location if inserting into partition that already exists (Sushanth Sowmyan, reviewed by Alan Gates)

Modified:
     hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java

Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1636814&r1=1636813&r2=1636814&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Wed Nov 5 07:43:40 2014
@@ -238,6 +238,8 @@ public class LoadSemanticAnalyzer extend

      // create final load/move work

+ boolean preservePartitionSpecs = false;
+
      Map<String, String> partSpec = ts.getPartSpec();
      if (partSpec == null) {
        partSpec = new LinkedHashMap<String, String>();
@@ -252,9 +254,14 @@ public class LoadSemanticAnalyzer extend
              throw new SemanticException(ErrorMsg.OFFLINE_TABLE_OR_PARTITION.
                  getMsg(ts.tableName + ":" + part.getName()));
            }
- outputs.add(new WriteEntity(part,
- (isOverWrite ? WriteEntity.WriteType.INSERT_OVERWRITE :
- WriteEntity.WriteType.INSERT)));
+ if (isOverWrite){
+ outputs.add(new WriteEntity(part, WriteEntity.WriteType.INSERT_OVERWRITE));
+ } else {
+ outputs.add(new WriteEntity(part, WriteEntity.WriteType.INSERT));
+ // If partition already exists and we aren't overwriting it, then respect
+ // its current location info rather than picking it from the parent TableDesc
+ preservePartitionSpecs = true;
+ }
          } else {
            outputs.add(new WriteEntity(ts.tableHandle,
            (isOverWrite ? WriteEntity.WriteType.INSERT_OVERWRITE :
@@ -269,6 +276,12 @@ public class LoadSemanticAnalyzer extend
      LoadTableDesc loadTableWork;
      loadTableWork = new LoadTableDesc(new Path(fromURI),
        Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+ if (preservePartitionSpecs){
+ // Note : preservePartitionSpecs=true implies inheritTableSpecs=false but
+ // but preservePartitionSpecs=false(default) here is not sufficient enough
+ // info to set inheritTableSpecs=true
+ loadTableWork.setInheritTableSpecs(false);
+ }

      Task<? extends Serializable> childTask = TaskFactory.get(new MoveWork(getInputs(),
          getOutputs(), loadTableWork, null, true, isLocal), conf);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 5, '14 at 7:44a
activeNov 5, '14 at 7:44a
posts1
users1
websitehive.apache.org

1 user in discussion

Khorgath: 1 post

People

Translate

site design / logo © 2021 Grokbase