Grokbase Groups Hive commits May 2015
FAQ
Repository: hive
Updated Branches:
   refs/heads/master 95929308b -> 0d93438a3


HIVE-10776 - Schema on insert for bucketed tables throwing NullPointerException(Eugene Koifman, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d93438a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d93438a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d93438a

Branch: refs/heads/master
Commit: 0d93438a3543cb64cbe2ebcdc21e5b40c1dd86e6
Parents: 9592930
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri May 22 13:50:40 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Fri May 22 13:50:40 2015 -0700

----------------------------------------------------------------------
  .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 +-
  .../clientpositive/insert_into_with_schema2.q | 23 +++++
  .../insert_into_with_schema2.q.out | 98 ++++++++++++++++++++
  3 files changed, 126 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 4346f60..7f355e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -3910,7 +3910,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     * @throws SemanticException
     */
    private void handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest,
- RowResolver out_rwsch, RowResolver inputRR, QB qb,
+ RowResolver outputRR, RowResolver inputRR, QB qb,
                                           ASTNode selExprList) throws SemanticException {
      //(z,x)
      List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query
@@ -3932,7 +3932,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
      Map<String, ColumnInfo> targetCol2ColumnInfo = new HashMap<String, ColumnInfo>();
      int colListPos = 0;
      for(String targetCol : targetTableSchema) {
- targetCol2ColumnInfo.put(targetCol, out_rwsch.getColumnInfos().get(colListPos));
+ targetCol2ColumnInfo.put(targetCol, outputRR.getColumnInfos().get(colListPos));
        targetCol2Projection.put(targetCol, col_list.get(colListPos++));
      }
      Table target = qb.getMetaData().getDestTableForAlias(dest);
@@ -3976,16 +3976,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
          t.setText("TOK_NULL");
          ExprNodeDesc exp = genExprNodeDesc(new ASTNode(t), inputRR, tcCtx);
          new_col_list.add(exp);
- final String tableAlias = "";//is this OK? this column doesn't come from any table
+ final String tableAlias = null;//this column doesn't come from any table
          ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos),
            exp.getWritableObjectInspector(), tableAlias, false);
          newSchema.add(colInfo);
+ outputRR.addMappingOnly(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo);
        }
        colListPos++;
      }
      col_list.clear();
      col_list.addAll(new_col_list);
- out_rwsch.setRowSchema(new RowSchema(newSchema));
+ outputRR.setRowSchema(new RowSchema(newSchema));
    }
    String recommendName(ExprNodeDesc exp, String colAlias) {
      if (!colAlias.startsWith(autogenColAliasPrfxLbl)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
new file mode 100644
index 0000000..b7c6b58
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
@@ -0,0 +1,23 @@
+-- SORT_QUERY_RESULTS;
+
+set hive.enforce.bucketing=true;
+
+create table studenttab10k (age2 int);
+insert into studenttab10k values(1);
+
+create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets;
+
+insert into student_acid(age) select * from studenttab10k;
+
+select * from student_acid;
+
+insert into student_acid(grade, age) select 3 g, * from studenttab10k;
+
+select * from student_acid;
+
+insert into student_acid(grade, age) values(20, 2);
+
+insert into student_acid(age) values(22);
+
+select * from student_acid;

http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
new file mode 100644
index 0000000..a55a82f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
@@ -0,0 +1,98 @@
+PREHOOK: query: create table studenttab10k (age2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: create table studenttab10k (age2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: insert into studenttab10k values(1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: insert into studenttab10k values(1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@studenttab10k
+POSTHOOK: Lineage: studenttab10k.age2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@student_acid
+PREHOOK: query: insert into student_acid(age) select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(age) select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2, type:int, comment:null), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1 NULL
+PREHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2, type:int, comment:null), ]
+POSTHOOK: Lineage: student_acid.grade SIMPLE []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1 3
+1 NULL
+PREHOOK: query: insert into student_acid(grade, age) values(20, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(grade, age) values(20, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: insert into student_acid(age) values(22)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__3
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(age) values(22)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__3
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1 3
+1 NULL
+2 20
+22 NULL

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 22, '15 at 8:50p
activeMay 22, '15 at 8:50p
posts1
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 1 post

People

Translate

site design / logo © 2021 Grokbase