FAQ
Repository: hive
Updated Branches:
   refs/heads/hbase-metastore afe1b5e3a -> 1a64664ae


HIVE-10875 : Select query with view in subquery adds underlying table as direct input (Thejas Nair, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2b8d5474
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2b8d5474
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2b8d5474

Branch: refs/heads/hbase-metastore
Commit: 2b8d5474e2c34f8a4a69172f1ef5f05da6299760
Parents: fac9ee9
Author: Thejas Nair <thejas@hortonworks.com>
Authored: Mon Jun 1 08:41:28 2015 -0700
Committer: Thejas Nair <thejas@hortonworks.com>
Committed: Mon Jun 1 08:41:48 2015 -0700

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/plan/PlanUtils.java | 6 ++-
  .../hadoop/hive/ql/plan/TestViewEntity.java | 53 ++++++++++++++++----
  .../clientpositive/authorization_view_sqlstd.q | 4 ++
  .../authorization_view_sqlstd.q.out | 14 ++++++
  4 files changed, 66 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2b8d5474/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index c8ae235..cb0b680 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -983,7 +983,11 @@ public final class PlanUtils {
        currentAlias = currentAlias.replace(SemanticAnalyzer.SUBQUERY_TAG_1, "")
            .replace(SemanticAnalyzer.SUBQUERY_TAG_2, "");
        ReadEntity input = viewAliasToInput.get(currentAlias);
- if (input == null) {
+ if (input == null && currentInput != null) {
+ // To handle the case of - select * from (select * from V1) A;
+ // the currentInput != null check above is needed.
+ // the alias list that case would be A:V1:T. Lookup on A would return null,
+ // we need to go further to find the view inside it.
          return currentInput;
        }
        currentInput = input;

http://git-wip-us.apache.org/repos/asf/hive/blob/2b8d5474/ql/src/test/org/apache/hadoop/hive/ql/plan/TestViewEntity.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestViewEntity.java b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestViewEntity.java
index 17a4e06..e24208e 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestViewEntity.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestViewEntity.java
@@ -54,6 +54,7 @@ public class TestViewEntity {
    }

    private static Driver driver;
+ private final String NAME_PREFIX = "TestViewEntity5".toLowerCase();

    @BeforeClass
    public static void onetimeSetup() throws Exception {
@@ -79,30 +80,62 @@ public class TestViewEntity {
     */
    @Test
    public void testUnionView() throws Exception {
- int ret = driver.run("create table t1(id int)").getResponseCode();
+ String prefix = "tunionview" + NAME_PREFIX;
+ final String tab1 = prefix + "t1";
+ final String tab2 = prefix + "t2";
+ final String view1 = prefix + "v1";
+ int ret = driver.run("create table " + tab1 + "(id int)").getResponseCode();
      assertEquals("Checking command success", 0, ret);
- ret = driver.run("create table t2(id int)").getResponseCode();
+ ret = driver.run("create table " + tab2 + "(id int)").getResponseCode();
      assertEquals("Checking command success", 0, ret);
- ret = driver.run("create view v1 as select t.id from "
- + "(select t1.id from t1 union all select t2.id from t2) as t")
+ ret = driver.run("create view " + view1 + " as select t.id from "
+ + "(select " + tab1 + ".id from " + tab1 + " union all select " + tab2 + ".id from " + tab2 + ") as t")
          .getResponseCode();
      assertEquals("Checking command success", 0, ret);

- driver.compile("select * from v1");
+ driver.compile("select * from " + view1 );
      // view entity
- assertEquals("default@v1", CheckInputReadEntity.readEntities[0].getName());
+ assertEquals("default@" + view1, CheckInputReadEntity.readEntities[0].getName());

      // first table in union query with view as parent
- assertEquals("default@t1", CheckInputReadEntity.readEntities[1].getName());
- assertEquals("default@v1", CheckInputReadEntity.readEntities[1]
+ assertEquals("default@" + tab1, CheckInputReadEntity.readEntities[1].getName());
+ assertFalse("Table is not direct input", CheckInputReadEntity.readEntities[1].isDirect());
+ assertEquals("default@" + view1, CheckInputReadEntity.readEntities[1]
          .getParents()
          .iterator().next().getName());
      // second table in union query with view as parent
- assertEquals("default@t2", CheckInputReadEntity.readEntities[2].getName());
- assertEquals("default@v1", CheckInputReadEntity.readEntities[2]
+ assertEquals("default@" + tab2, CheckInputReadEntity.readEntities[2].getName());
+ assertFalse("Table is not direct input", CheckInputReadEntity.readEntities[2].isDirect());
+ assertEquals("default@" + view1, CheckInputReadEntity.readEntities[2]
          .getParents()
          .iterator().next().getName());

    }

+
+ /**
+ * Verify that the parent entities are captured correctly for view in subquery
+ * @throws Exception
+ */
+ @Test
+ public void testViewInSubQuery() throws Exception {
+ String prefix = "tvsubquery" + NAME_PREFIX;
+ final String tab1 = prefix + "t";
+ final String view1 = prefix + "v";
+
+ int ret = driver.run("create table " + tab1 + "(id int)").getResponseCode();
+ assertEquals("Checking command success", 0, ret);
+ ret = driver.run("create view " + view1 + " as select * from " + tab1).getResponseCode();
+ assertEquals("Checking command success", 0, ret);
+
+ driver.compile("select * from " + view1 );
+ // view entity
+ assertEquals("default@" + view1, CheckInputReadEntity.readEntities[0].getName());
+
+ // table as second read entity
+ assertEquals("default@" + tab1, CheckInputReadEntity.readEntities[1].getName());
+ assertFalse("Table is not direct input", CheckInputReadEntity.readEntities[1].isDirect());
+
+ }
+
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/2b8d5474/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q b/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q
index 85d3ca3..8467c16 100644
--- a/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q
+++ b/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q
@@ -34,6 +34,10 @@ set user.name=user2;
  explain authorization select * from vt1;
  select * from vt1;

+-- verify input objects required does not include table
+-- even if view is within a sub query
+select * from (select * from vt1) a;
+
  set user.name=user1;

  grant all on table vt2 to user user2;

http://git-wip-us.apache.org/repos/asf/hive/blob/2b8d5474/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out b/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out
index 54c4ce7..461490b 100644
--- a/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out
+++ b/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out
@@ -109,6 +109,20 @@ POSTHOOK: type: QUERY
  POSTHOOK: Input: default@t1
  POSTHOOK: Input: default@vt1
  #### A masked pattern was here ####
+PREHOOK: query: -- verify input objects required does not include table
+-- even if view is within a sub query
+select * from (select * from vt1) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@vt1
+#### A masked pattern was here ####
+POSTHOOK: query: -- verify input objects required does not include table
+-- even if view is within a sub query
+select * from (select * from vt1) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@vt1
+#### A masked pattern was here ####
  PREHOOK: query: grant all on table vt2 to user user2
  PREHOOK: type: GRANT_PRIVILEGE
  PREHOOK: Output: default@vt2

Search Discussions

  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10808 : Inner join on Null throwing Cast Exception (Naveen Gangam via Ashutosh Chauhan)

    Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cabb527f
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cabb527f
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cabb527f

    Branch: refs/heads/hbase-metastore
    Commit: cabb527fcc9f84f126b6c9eed62bac160cdafb30
    Parents: a9eaa20
    Author: Naveen Gangam <ngangam@cloudera.com>
    Authored: Fri May 22 19:49:00 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Jun 1 11:51:30 2015 -0700

    ----------------------------------------------------------------------
      serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/cabb527f/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java
    index 83f34ce..8ddcde2 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java
    @@ -97,7 +97,7 @@ public class NullStructSerDe extends AbstractSerDe {
          }

          public Category getCategory() {
    - return Category.PRIMITIVE;
    + return Category.STRUCT;
          }

          @Override
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10660 : Fix typo in Type.getType(TTypeId) exception message (Keegan Witt via Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/424f6505
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/424f6505
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/424f6505

    Branch: refs/heads/hbase-metastore
    Commit: 424f6505acad5b4146e43cfd658b5ead21b50896
    Parents: cabb527
    Author: Ashutosh Chauhan <hashutosh@apache.org>
    Authored: Mon Jun 1 11:57:41 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Jun 1 11:57:41 2015 -0700

    ----------------------------------------------------------------------
      service/src/java/org/apache/hive/service/cli/Type.java | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/424f6505/service/src/java/org/apache/hive/service/cli/Type.java
    ----------------------------------------------------------------------
    diff --git a/service/src/java/org/apache/hive/service/cli/Type.java b/service/src/java/org/apache/hive/service/cli/Type.java
    index a96d2ac..428d2ad 100644
    --- a/service/src/java/org/apache/hive/service/cli/Type.java
    +++ b/service/src/java/org/apache/hive/service/cli/Type.java
    @@ -149,7 +149,7 @@ public enum Type {
              return type;
            }
          }
    - throw new IllegalArgumentException("Unregonized Thrift TTypeId value: " + tType);
    + throw new IllegalArgumentException("Unrecognized Thrift TTypeId value: " + tType);
        }

        public static Type getType(String name) {
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10473 Spark client is recreated even spark configuration is not changed (Jimmy, reviewed by Szehon)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/769ccc1c
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/769ccc1c
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/769ccc1c

    Branch: refs/heads/hbase-metastore
    Commit: 769ccc1c2040ba2cf7c6381a5b31a0868c60cd62
    Parents: eb48ffd
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Fri Apr 24 12:37:29 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:02:34 2015 -0700

    ----------------------------------------------------------------------
      .../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 11 ++++++++---
      1 file changed, 8 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/769ccc1c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    index 6d22f11..d733d71 100644
    --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    @@ -79,7 +79,7 @@ public class HiveConf extends Configuration {
        private final List<String> restrictList = new ArrayList<String>();

        private Pattern modWhiteListPattern = null;
    - private boolean isSparkConfigUpdated = false;
    + private volatile boolean isSparkConfigUpdated = false;

        public boolean getSparkConfigUpdated() {
          return isSparkConfigUpdated;
    @@ -2420,8 +2420,13 @@ public class HiveConf extends Configuration {
            throw new IllegalArgumentException("Cannot modify " + name + " at runtime. It is in the list"
                + "of parameters that can't be modified at runtime");
          }
    - isSparkConfigUpdated = isSparkRelatedConfig(name);
    - set(name, value);
    + String oldValue = name != null ? get(name) : null;
    + if (name == null || value == null || !value.equals(oldValue)) {
    + // When either name or value is null, the set method below will fail,
    + // and throw IllegalArgumentException
    + set(name, value);
    + isSparkConfigUpdated = isSparkRelatedConfig(name);
    + }
        }

        /**
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10869 : fold_case.q failing on trunk (Ashutosh Chauhan via Vikram Dixit)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6f005071
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6f005071
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6f005071

    Branch: refs/heads/hbase-metastore
    Commit: 6f00507136319f322616d428c70ffc4b72745ab0
    Parents: 8f4abd1
    Author: Ashutosh Chauhan <hashutosh@apache.org>
    Authored: Mon Jun 1 13:21:12 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Jun 1 13:21:12 2015 -0700

    ----------------------------------------------------------------------
      ql/src/test/results/clientpositive/fold_case.q.out | 8 ++++----
      1 file changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/6f005071/ql/src/test/results/clientpositive/fold_case.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/fold_case.q.out b/ql/src/test/results/clientpositive/fold_case.q.out
    index a6faca0..a9214f3 100644
    --- a/ql/src/test/results/clientpositive/fold_case.q.out
    +++ b/ql/src/test/results/clientpositive/fold_case.q.out
    @@ -267,7 +267,7 @@ STAGE PLANS:
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
                  Filter Operator
                    predicate: false (type: boolean)
    - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
                    Group By Operator
                      aggregations: count(1)
                      mode: hash
    @@ -405,14 +405,14 @@ STAGE PLANS:
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
                  Filter Operator
                    predicate: false (type: boolean)
    - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
                    Select Operator
                      expressions: null (type: void)
                      outputColumnNames: _col0
    - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
                      File Output Operator
                        compressed: false
    - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
                        table:
                            input format: org.apache.hadoop.mapred.TextInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10476 - Hive query should fail when it fails to initialize a session in SetSparkReducerParallelism [Spark Branch] (Chao, reviewed by Rui)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cee1183d
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cee1183d
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cee1183d

    Branch: refs/heads/hbase-metastore
    Commit: cee1183d3044c4fa0a7882c3499d59721f1b5492
    Parents: 88dfd27
    Author: Chao Sun <sunchao@apache.org>
    Authored: Tue Apr 28 09:39:18 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:03:09 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java | 2 ++
      1 file changed, 2 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/cee1183d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    index dcfbc6b..f9ef474 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    @@ -115,6 +115,8 @@ public class SetSparkReducerParallelism implements NodeProcessor {
                  sparkSession = SparkUtilities.getSparkSession(
                    context.getConf(), sparkSessionManager);
                  sparkMemoryAndCores = sparkSession.getMemoryAndCores();
    + } catch (HiveException e) {
    + throw new SemanticException("Failed to get a spark session: " + e);
                } catch (Exception e) {
                  LOG.warn("Failed to get spark memory/core info", e);
                } finally {
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-6727 : Table level stats for external tables are set incorrectly (Bing Li via Ashutosh Chauhan)

    Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f4abd1e
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f4abd1e
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f4abd1e

    Branch: refs/heads/hbase-metastore
    Commit: 8f4abd1e60b3410558e0a2cb6886334b7d505a98
    Parents: 424f650
    Author: Bing Li <sarah.libing@gmail.com>
    Authored: Mon Jun 1 01:35:00 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Jun 1 13:16:56 2015 -0700

    ----------------------------------------------------------------------
      .../src/java/org/apache/hadoop/hive/metastore/Warehouse.java | 2 +-
      .../encrypted/encryption_join_unencrypted_tbl.q.out | 8 ++++----
      .../test/results/clientpositive/spark/stats_noscan_2.q.out | 8 ++++----
      ql/src/test/results/clientpositive/stats_noscan_2.q.out | 8 ++++----
      4 files changed, 13 insertions(+), 13 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/8f4abd1e/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    index fc0e738..011d4d8 100755
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    @@ -525,7 +525,7 @@ public class Warehouse {
         */
        public FileStatus[] getFileStatusesForUnpartitionedTable(Database db, Table table)
            throws MetaException {
    - Path tablePath = getTablePath(db, table.getTableName());
    + Path tablePath = getDnsPath(new Path(table.getSd().getLocation()));
          try {
            FileSystem fileSys = tablePath.getFileSystem(conf);
            return HiveStatsUtils.getFileStatusRecurse(tablePath, -1, fileSys);

    http://git-wip-us.apache.org/repos/asf/hive/blob/8f4abd1e/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
    index 9b6f750..644c3ed 100644
    --- a/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
    +++ b/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
    @@ -628,13 +628,13 @@ STAGE PLANS:
                    columns.types int:string
      #### A masked pattern was here ####
                    name default.encrypted_table
    - numFiles 0
    + numFiles 1
                    numRows 0
                    rawDataSize 0
                    serialization.ddl struct encrypted_table { i32 key, string value}
                    serialization.format 1
                    serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    - totalSize 0
    + totalSize 5812
      #### A masked pattern was here ####
                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    @@ -648,13 +648,13 @@ STAGE PLANS:
                      columns.types int:string
      #### A masked pattern was here ####
                      name default.encrypted_table
    - numFiles 0
    + numFiles 1
                      numRows 0
                      rawDataSize 0
                      serialization.ddl struct encrypted_table { i32 key, string value}
                      serialization.format 1
                      serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    - totalSize 0
    + totalSize 5812
      #### A masked pattern was here ####
                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                    name: default.encrypted_table

    http://git-wip-us.apache.org/repos/asf/hive/blob/8f4abd1e/ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out b/ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out
    index 238727d..105c222 100644
    --- a/ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out
    +++ b/ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out
    @@ -54,10 +54,10 @@ Table Type: EXTERNAL_TABLE
      Table Parameters:
       COLUMN_STATS_ACCURATE true
       EXTERNAL TRUE
    - numFiles 0
    + numFiles 1
       numRows -1
       rawDataSize -1
    - totalSize 0
    + totalSize 11
      #### A masked pattern was here ####

      # Storage Information
    @@ -98,10 +98,10 @@ Table Type: EXTERNAL_TABLE
      Table Parameters:
       COLUMN_STATS_ACCURATE true
       EXTERNAL TRUE
    - numFiles 0
    + numFiles 1
       numRows 6
       rawDataSize 6
    - totalSize 0
    + totalSize 11
      #### A masked pattern was here ####

      # Storage Information

    http://git-wip-us.apache.org/repos/asf/hive/blob/8f4abd1e/ql/src/test/results/clientpositive/stats_noscan_2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/stats_noscan_2.q.out b/ql/src/test/results/clientpositive/stats_noscan_2.q.out
    index 238727d..105c222 100644
    --- a/ql/src/test/results/clientpositive/stats_noscan_2.q.out
    +++ b/ql/src/test/results/clientpositive/stats_noscan_2.q.out
    @@ -54,10 +54,10 @@ Table Type: EXTERNAL_TABLE
      Table Parameters:
       COLUMN_STATS_ACCURATE true
       EXTERNAL TRUE
    - numFiles 0
    + numFiles 1
       numRows -1
       rawDataSize -1
    - totalSize 0
    + totalSize 11
      #### A masked pattern was here ####

      # Storage Information
    @@ -98,10 +98,10 @@ Table Type: EXTERNAL_TABLE
      Table Parameters:
       COLUMN_STATS_ACCURATE true
       EXTERNAL TRUE
    - numFiles 0
    + numFiles 1
       numRows 6
       rawDataSize 6
    - totalSize 0
    + totalSize 11
      #### A masked pattern was here ####

      # Storage Information
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-8858: Visualize generated Spark plan [Spark Branch] (Chinna via Jimmy)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eb48ffdb
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eb48ffdb
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eb48ffdb

    Branch: refs/heads/hbase-metastore
    Commit: eb48ffdbfc0690480e2aeb53f03e249ad6091378
    Parents: 6f00507
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Wed Apr 22 14:51:35 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:02:18 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/spark/MapInput.java | 10 ++++
      .../hadoop/hive/ql/exec/spark/MapTran.java | 10 ++++
      .../hadoop/hive/ql/exec/spark/ReduceTran.java | 11 ++++
      .../hadoop/hive/ql/exec/spark/ShuffleTran.java | 10 ++++
      .../hadoop/hive/ql/exec/spark/SparkPlan.java | 58 ++++++++++++++++++++
      .../hadoop/hive/ql/exec/spark/SparkTran.java | 4 ++
      6 files changed, 103 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/eb48ffdb/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    index 8d18885..157e4d8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    @@ -85,4 +85,14 @@ public class MapInput implements SparkTran<WritableComparable, Writable,
          }

        }
    +
    + @Override
    + public String getName() {
    + return "MapInput";
    + }
    +
    + @Override
    + public Boolean isCacheEnable() {
    + return new Boolean(toCache);
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/eb48ffdb/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    index 638c387..f6a4d77 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    @@ -35,4 +35,14 @@ public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey,
          this.mapFunc = mapFunc;
        }

    + @Override
    + public String getName() {
    + return "MapTran";
    + }
    +
    + @Override
    + public Boolean isCacheEnable() {
    + // TODO Auto-generated method stub
    + return null;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/eb48ffdb/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    index dbc614b..fd6b31c 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    @@ -34,4 +34,15 @@ public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, H
        public void setReduceFunction(HiveReduceFunction redFunc) {
          this.reduceFunc = redFunc;
        }
    +
    + @Override
    + public String getName() {
    + return "Reduce";
    + }
    +
    + @Override
    + public Boolean isCacheEnable() {
    + // TODO Auto-generated method stub
    + return null;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/eb48ffdb/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    index 4a597ee..6cdab20 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    @@ -49,4 +49,14 @@ public class ShuffleTran implements SparkTran<HiveKey, BytesWritable, HiveKey, I
          }
          return result;
        }
    +
    + @Override
    + public String getName() {
    + return "Shuffle";
    + }
    +
    + @Override
    + public Boolean isCacheEnable() {
    + return new Boolean(toCache);
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/eb48ffdb/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    index b45494d..81b7e85 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    @@ -26,6 +26,8 @@ import java.util.List;
      import java.util.Map;
      import java.util.Set;

    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.hive.ql.io.HiveKey;
      import org.apache.hadoop.hive.ql.log.PerfLogger;
      import org.apache.hadoop.io.BytesWritable;
    @@ -36,6 +38,7 @@ import com.google.common.base.Preconditions;
      @SuppressWarnings("rawtypes")
      public class SparkPlan {
        private static final String CLASS_NAME = SparkPlan.class.getName();
    + private static final Log LOG = LogFactory.getLog(SparkPlan.class);
        private final PerfLogger perfLogger = PerfLogger.getPerfLogger();

        private final Set<SparkTran> rootTrans = new HashSet<SparkTran>();
    @@ -72,6 +75,8 @@ public class SparkPlan {
            tranToOutputRDDMap.put(tran, rdd);
          }

    + logSparkPlan();
    +
          JavaPairRDD<HiveKey, BytesWritable> finalRDD = null;
          for (SparkTran leafTran : leafTrans) {
            JavaPairRDD<HiveKey, BytesWritable> rdd = tranToOutputRDDMap.get(leafTran);
    @@ -86,6 +91,59 @@ public class SparkPlan {
          return finalRDD;
        }

    + private void logSparkPlan() {
    + LOG.info("------------------------------ Spark Plan -----------------------------");
    + Set<SparkTran> keySet = invertedTransGraph.keySet();
    + for (SparkTran sparkTran : keySet) {
    + if (sparkTran instanceof ReduceTran) {
    + String sparkPlan = " " + sparkTran.getName();
    + sparkPlan = getSparkPlan(sparkTran, sparkPlan);
    + LOG.info(sparkPlan);
    + }
    + }
    + LOG.info("------------------------------ Spark Plan -----------------------------");
    + }
    +
    + private String getSparkPlan(SparkTran leaf, String sparkPlanMsg) {
    + if (leaf != null) {
    + List<SparkTran> parents = getParents(leaf);
    + if (parents.size() > 0) {
    + sparkPlanMsg = sparkPlanMsg + " <-- ";
    + boolean isFirst = true;
    + SparkTran parent = null;
    + for (SparkTran sparkTran : parents) {
    + if (isFirst) {
    + sparkPlanMsg = sparkPlanMsg + "( " + sparkTran.getName();
    + sparkPlanMsg = logCacheStatus(sparkPlanMsg, sparkTran);
    + isFirst = false;
    + } else {
    + sparkPlanMsg = sparkPlanMsg + "," + sparkTran.getName();
    + sparkPlanMsg = logCacheStatus(sparkPlanMsg, sparkTran);
    + }
    + if (getParents(sparkTran).size() > 0 && !(sparkTran instanceof ReduceTran)) {
    + parent = sparkTran;
    + }
    + }
    + sparkPlanMsg = sparkPlanMsg + " ) ";
    + return getSparkPlan(parent, sparkPlanMsg);
    + } else {
    + return sparkPlanMsg;
    + }
    + }
    + return sparkPlanMsg;
    + }
    +
    + private String logCacheStatus(String sparkPlanMsg, SparkTran sparkTran) {
    + if (sparkTran.isCacheEnable() != null) {
    + if (sparkTran.isCacheEnable().booleanValue()) {
    + sparkPlanMsg = sparkPlanMsg + " (cache on) ";
    + } else {
    + sparkPlanMsg = sparkPlanMsg + " (cache off) ";
    + }
    + }
    + return sparkPlanMsg;
    + }
    +
        public void addTran(SparkTran tran) {
          rootTrans.add(tran);
          leafTrans.add(tran);

    http://git-wip-us.apache.org/repos/asf/hive/blob/eb48ffdb/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    index 4daa61e..c3c48a0 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    @@ -25,4 +25,8 @@ import org.apache.spark.api.java.JavaPairRDD;
      public interface SparkTran<KI extends WritableComparable, VI, KO extends WritableComparable, VO> {
        JavaPairRDD<KO, VO> transform(
            JavaPairRDD<KI, VI> input);
    +
    + public String getName();
    +
    + public Boolean isCacheEnable();
      }
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10721 SparkSessionManagerImpl leaks SparkSessions [Spark Branch] (Jimmy reviewed by Xuefu, Chengxiang)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b3214f99
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b3214f99
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b3214f99

    Branch: refs/heads/hbase-metastore
    Commit: b3214f996c9889035110ba548ea8eddf8390cdce
    Parents: 3f27c6e
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Fri May 15 08:34:38 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:04:09 2015 -0700

    ----------------------------------------------------------------------
      .../ql/exec/spark/LocalHiveSparkClient.java | 8 ++-
      .../ql/exec/spark/RemoteHiveSparkClient.java | 6 +-
      .../ql/exec/spark/session/SparkSessionImpl.java | 2 +-
      .../spark/session/SparkSessionManagerImpl.java | 63 ++++++++++----------
      .../hive/spark/client/SparkClientImpl.java | 2 +-
      5 files changed, 43 insertions(+), 38 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
    index 7e33a3f..19d3fee 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
    @@ -199,7 +199,11 @@ public class LocalHiveSparkClient implements HiveSparkClient {

        @Override
        public void close() {
    - sc.stop();
    - client = null;
    + synchronized (LocalHiveSparkClient.class) {
    + client = null;
    + }
    + if (sc != null) {
    + sc.stop();
    + }
        }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    index bae30f3..8b15099 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    @@ -199,15 +199,19 @@ public class RemoteHiveSparkClient implements HiveSparkClient {

        @Override
        public void close() {
    - remoteClient.stop();
    + if (remoteClient != null) {
    + remoteClient.stop();
    + }
        }

        private static class JobStatusJob implements Job<Serializable> {

    + private static final long serialVersionUID = 1L;
          private final byte[] jobConfBytes;
          private final byte[] scratchDirBytes;
          private final byte[] sparkWorkBytes;

    + @SuppressWarnings("unused")
          private JobStatusJob() {
            // For deserialization.
            this(null, null, null);

    http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
    index 603f1ca..49e5f6c 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
    @@ -53,7 +53,7 @@ public class SparkSessionImpl implements SparkSession {
          isOpen = true;
          try {
            hiveSparkClient = HiveSparkClientFactory.createHiveSparkClient(conf);
    - } catch (Exception e) {
    + } catch (Throwable e) {
            throw new HiveException("Failed to create spark client.", e);
          }
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
    index ad012b6..616807c 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
    @@ -17,23 +17,19 @@
       */
      package org.apache.hadoop.hive.ql.exec.spark.session;

    -import com.google.common.base.Preconditions;
    -import org.apache.commons.logging.Log;
    -import org.apache.commons.logging.LogFactory;
    -import org.apache.hadoop.hive.conf.HiveConf;
    -import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory;
    -import org.apache.hadoop.hive.ql.metadata.HiveException;
    -import org.apache.hadoop.hive.shims.Utils;
    -import org.apache.hadoop.security.UserGroupInformation;
    -import org.apache.hive.spark.client.SparkClientFactory;
    -
      import java.io.IOException;
      import java.util.Collections;
      import java.util.HashSet;
      import java.util.Iterator;
      import java.util.Map;
      import java.util.Set;
    -import java.util.concurrent.atomic.AtomicBoolean;
    +
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hive.spark.client.SparkClientFactory;

      /**
       * Simple implementation of <i>SparkSessionManager</i>
    @@ -44,8 +40,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
      public class SparkSessionManagerImpl implements SparkSessionManager {
        private static final Log LOG = LogFactory.getLog(SparkSessionManagerImpl.class);

    - private Set<SparkSession> createdSessions;
    - private AtomicBoolean inited = new AtomicBoolean(false);
    + private Set<SparkSession> createdSessions = Collections.synchronizedSet(new HashSet<SparkSession>());
    + private volatile boolean inited = false;

        private static SparkSessionManagerImpl instance;

    @@ -78,14 +74,18 @@ public class SparkSessionManagerImpl implements SparkSessionManager {

        @Override
        public void setup(HiveConf hiveConf) throws HiveException {
    - if (inited.compareAndSet(false, true)) {
    - LOG.info("Setting up the session manager.");
    - createdSessions = Collections.synchronizedSet(new HashSet<SparkSession>());
    - Map<String, String> conf = HiveSparkClientFactory.initiateSparkConf(hiveConf);
    - try {
    - SparkClientFactory.initialize(conf);
    - } catch (IOException e) {
    - throw new HiveException("Error initializing SparkClientFactory", e);
    + if (!inited) {
    + synchronized (this) {
    + if (!inited) {
    + LOG.info("Setting up the session manager.");
    + Map<String, String> conf = HiveSparkClientFactory.initiateSparkConf(hiveConf);
    + try {
    + SparkClientFactory.initialize(conf);
    + inited = true;
    + } catch (IOException e) {
    + throw new HiveException("Error initializing SparkClientFactory", e);
    + }
    + }
            }
          }
        }
    @@ -104,14 +104,12 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
          if (existingSession != null) {
            // Open the session if it is closed.
            if (!existingSession.isOpen() && doOpen) {
    - existingSession.open(conf);
    + existingSession.open(conf);
            }
            return existingSession;
          }

          SparkSession sparkSession = new SparkSessionImpl();
    - createdSessions.add(sparkSession);
    -
          if (doOpen) {
            sparkSession.open(conf);
          }
    @@ -119,6 +117,7 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
          if (LOG.isDebugEnabled()) {
            LOG.debug(String.format("New session (%s) is created.", sparkSession.getSessionId()));
          }
    + createdSessions.add(sparkSession);
          return sparkSession;
        }

    @@ -144,17 +143,15 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
        @Override
        public void shutdown() {
          LOG.info("Closing the session manager.");
    - if (createdSessions != null) {
    - synchronized (createdSessions) {
    - Iterator<SparkSession> it = createdSessions.iterator();
    - while (it.hasNext()) {
    - SparkSession session = it.next();
    - session.close();
    - }
    - createdSessions.clear();
    + synchronized (createdSessions) {
    + Iterator<SparkSession> it = createdSessions.iterator();
    + while (it.hasNext()) {
    + SparkSession session = it.next();
    + session.close();
            }
    + createdSessions.clear();
          }
    - inited.set(false);
    + inited = false;
          SparkClientFactory.stop();
        }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    index 1bcd221..9e34a49 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    @@ -97,7 +97,7 @@ class SparkClientImpl implements SparkClient {
          try {
            // The RPC server will take care of timeouts here.
            this.driverRpc = rpcServer.registerClient(clientId, secret, protocol).get();
    - } catch (Exception e) {
    + } catch (Throwable e) {
            LOG.warn("Error while waiting for client to connect.", e);
            driverThread.interrupt();
            try {
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10527: NPE in SparkUtilities::isDedicatedCluster [Spark Branch] (Rui reviewed by Jimmy)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/81e8e665
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/81e8e665
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/81e8e665

    Branch: refs/heads/hbase-metastore
    Commit: 81e8e665247f8bff927cb206b5bb178455047d73
    Parents: cee1183
    Author: Rui Li <rui.li@intel.com>
    Authored: Thu Apr 30 10:15:34 2015 +0800
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:03:30 2015 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java | 1 +
      1 file changed, 1 insertion(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/81e8e665/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
    index 058d91d..21398d8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
    @@ -112,6 +112,7 @@ public class HiveSparkClientFactory {
          String sparkMaster = hiveConf.get("spark.master");
          if (sparkMaster == null) {
            sparkMaster = sparkConf.get("spark.master");
    + hiveConf.set("spark.master", sparkMaster);
          }
          if (sparkMaster.equals("yarn-cluster")) {
            sparkConf.put("spark.yarn.maxAppAttempts", "1");
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10458: Enable parallel order by for spark [Spark Branch] (Rui reviewed by Xuefu)

    Signed-off-by: Rui Li <rui.li@intel.com>


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37cccbc5
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37cccbc5
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37cccbc5

    Branch: refs/heads/hbase-metastore
    Commit: 37cccbc58fdee764743bda6ba2819e94e8d9a83f
    Parents: b3214f9
    Author: Rui Li <rui.li@intel.com>
    Authored: Thu May 21 10:13:17 2015 +0800
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:04:24 2015 -0700

    ----------------------------------------------------------------------
      .../correlation/ReduceSinkDeDuplication.java | 12 ++++-----
      .../spark/SetSparkReducerParallelism.java | 28 +++++++++++++++++++-
      .../hive/ql/parse/spark/GenSparkUtils.java | 2 +-
      .../hadoop/hive/ql/plan/ReduceSinkDesc.java | 15 +++++------
      .../clientpositive/spark/parallel_orderby.q.out | 6 ++---
      5 files changed, 44 insertions(+), 19 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
    index 404b759..8ac9ca7 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
    @@ -489,7 +489,7 @@ public class ReduceSinkDeDuplication implements Transform {
            if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
              CorrelationUtilities.replaceReduceSinkWithSelectOperator(
                  cRS, dedupCtx.getPctx(), dedupCtx);
    - pRS.getConf().setEnforceSort(true);
    + pRS.getConf().setDeduplicated(true);
              return true;
            }
            return false;
    @@ -512,7 +512,7 @@ public class ReduceSinkDeDuplication implements Transform {
            if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
              CorrelationUtilities.removeReduceSinkForGroupBy(
                  cRS, cGBY, dedupCtx.getPctx(), dedupCtx);
    - pRS.getConf().setEnforceSort(true);
    + pRS.getConf().setDeduplicated(true);
              return true;
            }
            return false;
    @@ -535,7 +535,7 @@ public class ReduceSinkDeDuplication implements Transform {
                  CorrelationUtilities.findPossibleParent(
                      pJoin, ReduceSinkOperator.class, dedupCtx.trustScript());
              if (pRS != null) {
    - pRS.getConf().setEnforceSort(true);
    + pRS.getConf().setDeduplicated(true);
              }
              return true;
            }
    @@ -559,7 +559,7 @@ public class ReduceSinkDeDuplication implements Transform {
                  CorrelationUtilities.findPossibleParent(
                      pJoin, ReduceSinkOperator.class, dedupCtx.trustScript());
              if (pRS != null) {
    - pRS.getConf().setEnforceSort(true);
    + pRS.getConf().setDeduplicated(true);
              }
              return true;
            }
    @@ -579,7 +579,7 @@ public class ReduceSinkDeDuplication implements Transform {
            if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
              CorrelationUtilities.replaceReduceSinkWithSelectOperator(
                  cRS, dedupCtx.getPctx(), dedupCtx);
    - pRS.getConf().setEnforceSort(true);
    + pRS.getConf().setDeduplicated(true);
              return true;
            }
            return false;
    @@ -596,7 +596,7 @@ public class ReduceSinkDeDuplication implements Transform {
                    start, ReduceSinkOperator.class, dedupCtx.trustScript());
            if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
              CorrelationUtilities.removeReduceSinkForGroupBy(cRS, cGBY, dedupCtx.getPctx(), dedupCtx);
    - pRS.getConf().setEnforceSort(true);
    + pRS.getConf().setDeduplicated(true);
              return true;
            }
            return false;

    http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    index f9ef474..5f9225c 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
    @@ -18,6 +18,7 @@

      package org.apache.hadoop.hive.ql.optimizer.spark;

    +import java.util.List;
      import java.util.Stack;

      import org.apache.commons.logging.Log;
    @@ -26,6 +27,7 @@ import org.apache.hadoop.hive.common.ObjectPair;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
      import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
    +import org.apache.hadoop.hive.ql.exec.LimitOperator;
      import org.apache.hadoop.hive.ql.exec.Operator;
      import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
      import org.apache.hadoop.hive.ql.exec.Utilities;
    @@ -75,7 +77,7 @@ public class SetSparkReducerParallelism implements NodeProcessor {

          context.getVisitedReduceSinks().add(sink);

    - if (desc.getNumReducers() <= 0) {
    + if (needSetParallelism(sink, context.getConf())) {
            if (constantReducers > 0) {
              LOG.info("Parallelism for reduce sink " + sink + " set by user to " + constantReducers);
              desc.setNumReducers(constantReducers);
    @@ -158,4 +160,28 @@ public class SetSparkReducerParallelism implements NodeProcessor {
          return false;
        }

    + // tests whether the RS needs automatic setting parallelism
    + private boolean needSetParallelism(ReduceSinkOperator reduceSink, HiveConf hiveConf) {
    + ReduceSinkDesc desc = reduceSink.getConf();
    + if (desc.getNumReducers() <= 0) {
    + return true;
    + }
    + if (desc.getNumReducers() == 1 && desc.hasOrderBy() &&
    + hiveConf.getBoolVar(HiveConf.ConfVars.HIVESAMPLINGFORORDERBY) && !desc.isDeduplicated()) {
    + List<Operator<? extends OperatorDesc>> children = reduceSink.getChildOperators();
    + while (children != null && children.size() > 0) {
    + if (children.size() != 1 || children.get(0) instanceof LimitOperator) {
    + return false;
    + }
    + if (children.get(0) instanceof ReduceSinkOperator ||
    + children.get(0) instanceof FileSinkOperator) {
    + break;
    + }
    + children = children.get(0).getChildOperators();
    + }
    + return true;
    + }
    + return false;
    + }
    +
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
    index e27ce0d..7992c88 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
    @@ -399,7 +399,7 @@ public class GenSparkUtils {
         */
        private static boolean groupByNeedParLevelOrder(ReduceSinkOperator reduceSinkOperator) {
          // whether we have to enforce sort anyway, e.g. in case of RS deduplication
    - if (reduceSinkOperator.getConf().isEnforceSort()) {
    + if (reduceSinkOperator.getConf().isDeduplicated()) {
            return true;
          }
          List<Operator<? extends OperatorDesc>> children = reduceSinkOperator.getChildOperators();

    http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
    index 1891dff..b4316ec 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
    @@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.plan;
      import java.util.ArrayList;
      import java.util.EnumSet;
      import java.util.List;
    -import java.util.Map.Entry;

      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    @@ -114,8 +113,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
        // Write type, since this needs to calculate buckets differently for updates and deletes
        private AcidUtils.Operation writeType;

    - // whether we'll enforce the sort order of the RS
    - private transient boolean enforceSort = false;
    + // whether this RS is deduplicated
    + private transient boolean isDeduplicated = false;

        // used by spark mode to decide whether global order is needed
        private transient boolean hasOrderBy = false;
    @@ -174,7 +173,7 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
          desc.setStatistics(this.getStatistics());
          desc.setSkipTag(skipTag);
          desc.reduceTraits = reduceTraits.clone();
    - desc.setEnforceSort(enforceSort);
    + desc.setDeduplicated(isDeduplicated);
          desc.setHasOrderBy(hasOrderBy);
          return desc;
        }
    @@ -434,12 +433,12 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
          return writeType;
        }

    - public boolean isEnforceSort() {
    - return enforceSort;
    + public boolean isDeduplicated() {
    + return isDeduplicated;
        }

    - public void setEnforceSort(boolean isDeduplicated) {
    - this.enforceSort = isDeduplicated;
    + public void setDeduplicated(boolean isDeduplicated) {
    + this.isDeduplicated = isDeduplicated;
        }

        public boolean hasOrderBy() {

    http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out b/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
    index 0194dbb..03314ea 100644
    --- a/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
    +++ b/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
    @@ -38,7 +38,7 @@ STAGE PLANS:
        Stage: Stage-1
          Spark
            Edges:
    - Reducer 2 <- Map 1 (SORT, 1)
    + Reducer 2 <- Map 1 (SORT, 4)
      #### A masked pattern was here ####
            Vertices:
              Map 1
    @@ -117,7 +117,7 @@ Retention: 0
      Table Type: MANAGED_TABLE
      Table Parameters:
       COLUMN_STATS_ACCURATE true
    - numFiles 1
    + numFiles 4
       numRows 48
       rawDataSize 512
       totalSize 560
    @@ -231,7 +231,7 @@ Retention: 0
      Table Type: MANAGED_TABLE
      Table Parameters:
       COLUMN_STATS_ACCURATE true
    - numFiles 1
    + numFiles 4
       numRows 48
       rawDataSize 512
       totalSize 560
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10626 Spark paln need to be updated [Spark Branch] (Chinna via Jimmy)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2df14f9b
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2df14f9b
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2df14f9b

    Branch: refs/heads/hbase-metastore
    Commit: 2df14f9b04dec968751f7b336bde7a5451af9d93
    Parents: 81e8e66
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Fri May 8 13:14:24 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:03:42 2015 -0700

    ----------------------------------------------------------------------
      .../hive/ql/exec/spark/GroupByShuffler.java | 4 +
      .../hadoop/hive/ql/exec/spark/MapInput.java | 8 +-
      .../hadoop/hive/ql/exec/spark/MapTran.java | 9 +-
      .../hadoop/hive/ql/exec/spark/ReduceTran.java | 9 +-
      .../hadoop/hive/ql/exec/spark/ShuffleTran.java | 16 +-
      .../hive/ql/exec/spark/SortByShuffler.java | 5 +
      .../hadoop/hive/ql/exec/spark/SparkPlan.java | 165 ++++++++++++++-----
      .../hive/ql/exec/spark/SparkShuffler.java | 2 +
      .../hadoop/hive/ql/exec/spark/SparkTran.java | 2 +
      9 files changed, 176 insertions(+), 44 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
    index b8e36cb..e128dd2 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
    @@ -33,4 +33,8 @@ public class GroupByShuffler implements SparkShuffler {
          return input.groupByKey();
        }

    + @Override
    + public String getName() {
    + return "GroupBy";
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    index 157e4d8..26cfebd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
    @@ -36,6 +36,7 @@ public class MapInput implements SparkTran<WritableComparable, Writable,
        private JavaPairRDD<WritableComparable, Writable> hadoopRDD;
        private boolean toCache;
        private final SparkPlan sparkPlan;
    + private String name = "MapInput";

        public MapInput(SparkPlan sparkPlan, JavaPairRDD<WritableComparable, Writable> hadoopRDD) {
          this(sparkPlan, hadoopRDD, false);
    @@ -88,11 +89,16 @@ public class MapInput implements SparkTran<WritableComparable, Writable,

        @Override
        public String getName() {
    - return "MapInput";
    + return name;
        }

        @Override
        public Boolean isCacheEnable() {
          return new Boolean(toCache);
        }
    +
    + @Override
    + public void setName(String name) {
    + this.name = name;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    index f6a4d77..2170243 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    @@ -24,6 +24,7 @@ import org.apache.spark.api.java.JavaPairRDD;

      public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey, BytesWritable> {
        private HiveMapFunction mapFunc;
    + private String name = "MapTran";

        @Override
        public JavaPairRDD<HiveKey, BytesWritable> transform(
    @@ -37,12 +38,16 @@ public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey,

        @Override
        public String getName() {
    - return "MapTran";
    + return name;
        }

        @Override
        public Boolean isCacheEnable() {
    - // TODO Auto-generated method stub
          return null;
        }
    +
    + @Override
    + public void setName(String name) {
    + this.name = name;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    index fd6b31c..e60dfac 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    @@ -24,6 +24,7 @@ import org.apache.spark.api.java.JavaPairRDD;

      public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, HiveKey, BytesWritable> {
        private HiveReduceFunction reduceFunc;
    + private String name = "Reduce";

        @Override
        public JavaPairRDD<HiveKey, BytesWritable> transform(
    @@ -37,12 +38,16 @@ public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, H

        @Override
        public String getName() {
    - return "Reduce";
    + return name;
        }

        @Override
        public Boolean isCacheEnable() {
    - // TODO Auto-generated method stub
          return null;
        }
    +
    + @Override
    + public void setName(String name) {
    + this.name = name;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    index 6cdab20..a774395 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
    @@ -28,6 +28,7 @@ public class ShuffleTran implements SparkTran<HiveKey, BytesWritable, HiveKey, I
        private final int numOfPartitions;
        private final boolean toCache;
        private final SparkPlan sparkPlan;
    + private String name = "Shuffle";

        public ShuffleTran(SparkPlan sparkPlan, SparkShuffler sf, int n) {
          this(sparkPlan, sf, n, false);
    @@ -50,13 +51,26 @@ public class ShuffleTran implements SparkTran<HiveKey, BytesWritable, HiveKey, I
          return result;
        }

    + public int getNoOfPartitions() {
    + return numOfPartitions;
    + }
    +
        @Override
        public String getName() {
    - return "Shuffle";
    + return name;
        }

        @Override
        public Boolean isCacheEnable() {
          return new Boolean(toCache);
        }
    +
    + @Override
    + public void setName(String name) {
    + this.name = name;
    + }
    +
    + public SparkShuffler getShuffler() {
    + return shuffler;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
    index 2545a9d..766813c 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
    @@ -56,6 +56,11 @@ public class SortByShuffler implements SparkShuffler {
          return rdd.mapPartitionsToPair(new ShuffleFunction());
        }

    + @Override
    + public String getName() {
    + return "SortBy";
    + }
    +
        private static class ShuffleFunction implements
            PairFlatMapFunction<Iterator<Tuple2<HiveKey, BytesWritable>>,
                HiveKey, Iterable<BytesWritable>> {

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    index 81b7e85..ee5c78a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    @@ -91,57 +91,146 @@ public class SparkPlan {
          return finalRDD;
        }

    + private void addNumberToTrans() {
    + int i = 1;
    + String name = null;
    +
    + // Traverse leafTran & transGraph add numbers to trans
    + for (SparkTran leaf : leafTrans) {
    + name = leaf.getName() + " " + i++;
    + leaf.setName(name);
    + }
    + Set<SparkTran> sparkTrans = transGraph.keySet();
    + for (SparkTran tran : sparkTrans) {
    + name = tran.getName() + " " + i++;
    + tran.setName(name);
    + }
    + }
    +
        private void logSparkPlan() {
    - LOG.info("------------------------------ Spark Plan -----------------------------");
    - Set<SparkTran> keySet = invertedTransGraph.keySet();
    - for (SparkTran sparkTran : keySet) {
    - if (sparkTran instanceof ReduceTran) {
    - String sparkPlan = " " + sparkTran.getName();
    - sparkPlan = getSparkPlan(sparkTran, sparkPlan);
    - LOG.info(sparkPlan);
    + addNumberToTrans();
    + ArrayList<SparkTran> leafTran = new ArrayList<SparkTran>();
    + leafTran.addAll(leafTrans);
    +
    + for (SparkTran leaf : leafTrans) {
    + collectLeafTrans(leaf, leafTran);
    + }
    +
    + // Start Traverse from the leafTrans and get parents of each leafTrans till
    + // the end
    + StringBuilder sparkPlan = new StringBuilder(
    + "\n\t!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Spark Plan !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! \n\n");
    + for (SparkTran leaf : leafTran) {
    + sparkPlan.append(leaf.getName());
    + getSparkPlan(leaf, sparkPlan);
    + sparkPlan.append("\n");
    + }
    + sparkPlan
    + .append(" \n\t!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Spark Plan !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ");
    + LOG.info(sparkPlan);
    + }
    +
    + private void collectLeafTrans(SparkTran leaf, List<SparkTran> reduceTrans) {
    + List<SparkTran> parents = getParents(leaf);
    + if (parents.size() > 0) {
    + SparkTran nextLeaf = null;
    + for (SparkTran leafTran : parents) {
    + if (leafTran instanceof ReduceTran) {
    + reduceTrans.add(leafTran);
    + } else {
    + if (getParents(leafTran).size() > 0)
    + nextLeaf = leafTran;
    + }
            }
    + if (nextLeaf != null)
    + collectLeafTrans(nextLeaf, reduceTrans);
          }
    - LOG.info("------------------------------ Spark Plan -----------------------------");
    - }
    -
    - private String getSparkPlan(SparkTran leaf, String sparkPlanMsg) {
    - if (leaf != null) {
    - List<SparkTran> parents = getParents(leaf);
    - if (parents.size() > 0) {
    - sparkPlanMsg = sparkPlanMsg + " <-- ";
    - boolean isFirst = true;
    - SparkTran parent = null;
    - for (SparkTran sparkTran : parents) {
    - if (isFirst) {
    - sparkPlanMsg = sparkPlanMsg + "( " + sparkTran.getName();
    - sparkPlanMsg = logCacheStatus(sparkPlanMsg, sparkTran);
    - isFirst = false;
    - } else {
    - sparkPlanMsg = sparkPlanMsg + "," + sparkTran.getName();
    - sparkPlanMsg = logCacheStatus(sparkPlanMsg, sparkTran);
    - }
    - if (getParents(sparkTran).size() > 0 && !(sparkTran instanceof ReduceTran)) {
    - parent = sparkTran;
    - }
    - }
    - sparkPlanMsg = sparkPlanMsg + " ) ";
    - return getSparkPlan(parent, sparkPlanMsg);
    + }
    +
    + private void getSparkPlan(SparkTran tran, StringBuilder sparkPlan) {
    + List<SparkTran> parents = getParents(tran);
    + List<SparkTran> nextLeaf = new ArrayList<SparkTran>();
    + if (parents.size() > 0) {
    + sparkPlan.append(" <-- ");
    + boolean isFirst = true;
    + for (SparkTran leaf : parents) {
    + if (isFirst) {
    + sparkPlan.append("( " + leaf.getName());
    + if (leaf instanceof ShuffleTran) {
    + logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
    + } else {
    + logCacheStatus(leaf, sparkPlan);
    + }
    + isFirst = false;
    + } else {
    + sparkPlan.append("," + leaf.getName());
    + if (leaf instanceof ShuffleTran) {
    + logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
    + } else {
    + logCacheStatus(leaf, sparkPlan);
    + }
    + }
    + // Leave reduceTran it will be expanded in the next line
    + if (getParents(leaf).size() > 0 && !(leaf instanceof ReduceTran)) {
    + nextLeaf.add(leaf);
    + }
    + }
    + sparkPlan.append(" ) ");
    + if (nextLeaf.size() > 1) {
    + logLeafTran(nextLeaf, sparkPlan);
            } else {
    - return sparkPlanMsg;
    + if (nextLeaf.size() != 0)
    + getSparkPlan(nextLeaf.get(0), sparkPlan);
            }
          }
    - return sparkPlanMsg;
        }

    - private String logCacheStatus(String sparkPlanMsg, SparkTran sparkTran) {
    + private void logLeafTran(List<SparkTran> parent, StringBuilder sparkPlan) {
    + sparkPlan.append(" <-- ");
    + boolean isFirst = true;
    + for (SparkTran sparkTran : parent) {
    + List<SparkTran> parents = getParents(sparkTran);
    + SparkTran leaf = parents.get(0);
    + if (isFirst) {
    + sparkPlan.append("( " + leaf.getName());
    + if (leaf instanceof ShuffleTran) {
    + logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
    + } else {
    + logCacheStatus(leaf, sparkPlan);
    + }
    + isFirst = false;
    + } else {
    + sparkPlan.append("," + leaf.getName());
    + if (leaf instanceof ShuffleTran) {
    + logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
    + } else {
    + logCacheStatus(leaf, sparkPlan);
    + }
    + }
    + }
    + sparkPlan.append(" ) ");
    + }
    +
    + private void logShuffleTranStatus(ShuffleTran leaf, StringBuilder sparkPlan) {
    + int noOfPartitions = leaf.getNoOfPartitions();
    + sparkPlan.append(" ( Partitions " + noOfPartitions);
    + SparkShuffler shuffler = leaf.getShuffler();
    + sparkPlan.append(", " + shuffler.getName());
    + if (leaf.isCacheEnable()) {
    + sparkPlan.append(", Cache on");
    + } else {
    + sparkPlan.append(", Cache off");
    + }
    + }
    +
    + private void logCacheStatus(SparkTran sparkTran, StringBuilder sparkPlan) {
          if (sparkTran.isCacheEnable() != null) {
            if (sparkTran.isCacheEnable().booleanValue()) {
    - sparkPlanMsg = sparkPlanMsg + " (cache on) ";
    + sparkPlan.append(" (cache on) ");
            } else {
    - sparkPlanMsg = sparkPlanMsg + " (cache off) ";
    + sparkPlan.append(" (cache off) ");
            }
          }
    - return sparkPlanMsg;
        }

        public void addTran(SparkTran tran) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
    index 53845a0..40e251f 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
    @@ -27,4 +27,6 @@ public interface SparkShuffler {
        JavaPairRDD<HiveKey, Iterable<BytesWritable>> shuffle(
            JavaPairRDD<HiveKey, BytesWritable> input, int numPartitions);

    + public String getName();
    +
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    index c3c48a0..671c983 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
    @@ -28,5 +28,7 @@ public interface SparkTran<KI extends WritableComparable, VI, KO extends Writabl

        public String getName();

    + public void setName(String name);
    +
        public Boolean isCacheEnable();
      }
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10302: Load small tables (for map join) in executor memory only once [Spark Branch], rebased with master (reviewed by Xuefu)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3629859
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3629859
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3629859

    Branch: refs/heads/hbase-metastore
    Commit: d3629859a2cdc3f45a83582db1e100cd9f3a482f
    Parents: 441d4c0
    Author: Xuefu Zhang <xzhang@cloudera.com>
    Authored: Mon Jun 1 14:09:08 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:09:08 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/MapJoinOperator.java | 6 ++++-
      .../hive/ql/exec/spark/HashTableLoader.java | 23 +++++++++++++++++---
      .../ql/exec/spark/HivePairFlatMapFunction.java | 1 +
      .../hive/ql/exec/spark/SparkUtilities.java | 6 +++++
      4 files changed, 32 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    index b1352f3..15cafdd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    @@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.conf.Configuration;
      import org.apache.hadoop.hive.common.ObjectPair;
      import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
      import org.apache.hadoop.hive.ql.HashTableLoaderFactory;
      import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
      import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
    @@ -50,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.Reusable
      import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
      import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
      import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
    +import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities;
      import org.apache.hadoop.hive.ql.io.HiveKey;
      import org.apache.hadoop.hive.ql.log.PerfLogger;
      import org.apache.hadoop.hive.ql.metadata.HiveException;
    @@ -521,7 +523,9 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem

          // in mapreduce case, we need to always clear up as mapreduce doesn't have object registry.
          if ((this.getExecContext() != null) && (this.getExecContext().getLocalWork() != null)
    - && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())) {
    + && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())
    + && !(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")
    + && SparkUtilities.isDedicatedCluster(hconf))) {
            if (isLogInfoEnabled) {
              LOG.info("MR: Clearing all map join table containers.");
            }

    http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
    index 043f1f7..1d674e9 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
    @@ -113,15 +113,32 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
              }
              String fileName = localWork.getBucketFileName(bigInputPath);
              Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte) pos, fileName);
    - LOG.info("\tLoad back all hashtable files from tmp folder uri:" + path);
    - mapJoinTables[pos] = mapJoinTableSerdes[pos].load(fs, path);
    + mapJoinTables[pos] = load(fs, path, mapJoinTableSerdes[pos]);
            }
          } catch (Exception e) {
            throw new HiveException(e);
          }
        }

    - @SuppressWarnings("unchecked")
    + private MapJoinTableContainer load(FileSystem fs, Path path,
    + MapJoinTableContainerSerDe mapJoinTableSerde) throws HiveException {
    + LOG.info("\tLoad back all hashtable files from tmp folder uri:" + path);
    + if (!SparkUtilities.isDedicatedCluster(hconf)) {
    + return mapJoinTableSerde.load(fs, path);
    + }
    + MapJoinTableContainer mapJoinTable = SmallTableCache.get(path);
    + if (mapJoinTable == null) {
    + synchronized (path.toString().intern()) {
    + mapJoinTable = SmallTableCache.get(path);
    + if (mapJoinTable == null) {
    + mapJoinTable = mapJoinTableSerde.load(fs, path);
    + SmallTableCache.cache(path, mapJoinTable);
    + }
    + }
    + }
    + return mapJoinTable;
    + }
    +
        private void loadDirectly(MapJoinTableContainer[] mapJoinTables, String inputFileName)
            throws Exception {
          MapredLocalWork localWork = context.getLocalWork();

    http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
    index 2f137f9..7df626b 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
    @@ -48,6 +48,7 @@ public abstract class HivePairFlatMapFunction<T, K, V> implements PairFlatMapFun
        protected void initJobConf() {
          if (jobConf == null) {
            jobConf = KryoSerializer.deserializeJobConf(this.buffer);
    + SmallTableCache.initialize(jobConf);
            setupMRLegacyConfigs();
          }
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    index ef612bd..ca0ffb6 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    @@ -25,6 +25,7 @@ import java.util.Collection;
      import java.util.UUID;

      import org.apache.commons.io.FilenameUtils;
    +import org.apache.hadoop.conf.Configuration;
      import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.hive.conf.HiveConf;
    @@ -108,6 +109,11 @@ public class SparkUtilities {
          return name;
        }

    + public static boolean isDedicatedCluster(Configuration conf) {
    + String master = conf.get("spark.master");
    + return master.startsWith("yarn-") || master.startsWith("local");
    + }
    +
        public static SparkSession getSparkSession(HiveConf conf,
            SparkSessionManager sparkSessionManager) throws HiveException {
          SparkSession sparkSession = SessionState.get().getSparkSession();
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10710 Delete GenericUDF.getConstantLongValue (Alexander Pivovarov, reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/55b28a0a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/55b28a0a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/55b28a0a

    Branch: refs/heads/hbase-metastore
    Commit: 55b28a0a93240270956688ec4bb5a98337bceb00
    Parents: d362985
    Author: Alexander Pivovarov <apivovarov@gmail.com>
    Authored: Thu May 14 10:12:06 2015 -0700
    Committer: Alexander Pivovarov <apivovarov@gmail.com>
    Committed: Mon Jun 1 14:35:48 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/udf/generic/GenericUDF.java | 23 --------------------
      1 file changed, 23 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/55b28a0a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    index b043bdc..e8df5d3 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    @@ -538,29 +538,6 @@ public abstract class GenericUDF implements Closeable {
          return v;
        }

    - protected Long getConstantLongValue(ObjectInspector[] arguments, int i)
    - throws UDFArgumentTypeException {
    - Object constValue = ((ConstantObjectInspector) arguments[i]).getWritableConstantValue();
    - if (constValue == null) {
    - return null;
    - }
    - long v;
    - if (constValue instanceof IntWritable) {
    - v = ((LongWritable) constValue).get();
    - } else if (constValue instanceof IntWritable) {
    - v = ((IntWritable) constValue).get();
    - } else if (constValue instanceof ShortWritable) {
    - v = ((ShortWritable) constValue).get();
    - } else if (constValue instanceof ByteWritable) {
    - v = ((ByteWritable) constValue).get();
    - } else {
    - throw new UDFArgumentTypeException(i, getFuncName()
    - + " only takes LONG/INT/SHORT/BYTE types as " + getArgOrder(i) + " argument, got "
    - + constValue.getClass());
    - }
    - return v;
    - }
    -
        protected String getArgOrder(int i) {
          i++;
          switch (i % 100) {
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10434 - Cancel connection when remote Spark driver process has failed [Spark Branch] (Chao, reviewed by Marcelo)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/88dfd274
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/88dfd274
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/88dfd274

    Branch: refs/heads/hbase-metastore
    Commit: 88dfd274f23bb3941a7c64e6ed932336e6c84890
    Parents: 769ccc1
    Author: Chao Sun <sunchao@apache.org>
    Authored: Fri Apr 24 15:16:36 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:02:51 2015 -0700

    ----------------------------------------------------------------------
      .../apache/hive/spark/client/SparkClientImpl.java | 3 ++-
      .../apache/hive/spark/client/rpc/RpcServer.java | 18 ++++++++++++++++++
      2 files changed, 20 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/88dfd274/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    index 71e432d..1bcd221 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    @@ -180,7 +180,7 @@ class SparkClientImpl implements SparkClient {
          protocol.cancel(jobId);
        }

    - private Thread startDriver(RpcServer rpcServer, final String clientId, final String secret)
    + private Thread startDriver(final RpcServer rpcServer, final String clientId, final String secret)
            throws IOException {
          Runnable runnable;
          final String serverAddress = rpcServer.getAddress();
    @@ -424,6 +424,7 @@ class SparkClientImpl implements SparkClient {
                try {
                  int exitCode = child.waitFor();
                  if (exitCode != 0) {
    + rpcServer.cancelClient(clientId, "Child process exited before connecting back");
                    LOG.warn("Child process exited with code {}.", exitCode);
                  }
                } catch (InterruptedException ie) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/88dfd274/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcServer.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcServer.java b/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcServer.java
    index 32d4c46..68ee627 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcServer.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcServer.java
    @@ -164,6 +164,24 @@ public class RpcServer implements Closeable {
        }

        /**
    + * Tells the RPC server to cancel the connection from an existing pending client
    + * @param clientId The identifier for the client
    + * @param msg The error message about why the connection should be canceled
    + */
    + public void cancelClient(final String clientId, final String msg) {
    + final ClientInfo cinfo = pendingClients.remove(clientId);
    + if (cinfo == null) {
    + // Nothing to be done here.
    + return;
    + }
    + cinfo.timeoutFuture.cancel(true);
    + if (!cinfo.promise.isDone()) {
    + cinfo.promise.setFailure(new RuntimeException(
    + String.format("Cancel client '%s'. Error: " + msg, clientId)));
    + }
    + }
    +
    + /**
         * Creates a secret for identifying a client connection.
         */
        public String createSecret() {
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10877 : - adding missing UtilsForTest from previous patch


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b6b638a8
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b6b638a8
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b6b638a8

    Branch: refs/heads/hbase-metastore
    Commit: b6b638a86d33077e42d71ab22ad36d117a79c369
    Parents: 9bc8ea2
    Author: Thejas Nair <thejas@hortonworks.com>
    Authored: Mon Jun 1 16:41:13 2015 -0700
    Committer: Thejas Nair <thejas@hortonworks.com>
    Committed: Mon Jun 1 16:41:21 2015 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hive/UtilsForTest.java | 41 ++++++++++++++++++++
      1 file changed, 41 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/b6b638a8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java
    new file mode 100644
    index 0000000..699d446
    --- /dev/null
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java
    @@ -0,0 +1,41 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.hadoop.hive;
    +
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
    +
    +/**
    + * Test utilities
    + */
    +public class UtilsForTest {
    + /**
    + * Use this if you want a fresh metastore for your test, without any existing entries.
    + * It updates the configuration to point to new derby dir location
    + * @param conf HiveConf to be updated
    + * @param newloc new location within test temp dir for the metastore db
    + */
    + public static void setNewDerbyDbLocation(HiveConf conf, String newloc) {
    + String newDbLoc = System.getProperty("test.tmp.dir") + newloc + "metastore_db";
    +
    + conf.setVar(ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:;databaseName=" + newDbLoc
    + + ";create=true");
    + }
    +
    +}
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10877 : TestUtil class name confuses ptest2 (Thejas Nair, reviewed by Hari Sankar Sivarama Subramaniyan )


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3817bb53
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3817bb53
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3817bb53

    Branch: refs/heads/hbase-metastore
    Commit: 3817bb5363948c74da99c15c65c4935cc7356f08
    Parents: 55b28a0
    Author: Thejas Nair <thejas@hortonworks.com>
    Authored: Mon Jun 1 15:47:25 2015 -0700
    Committer: Thejas Nair <thejas@hortonworks.com>
    Committed: Mon Jun 1 15:47:25 2015 -0700

    ----------------------------------------------------------------------
      .../java/org/apache/hadoop/hive/TestUtil.java | 41 --------------------
      .../plugin/TestHiveAuthorizerShowFilters.java | 4 +-
      2 files changed, 2 insertions(+), 43 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/3817bb53/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java
    deleted file mode 100644
    index 304fe5b..0000000
    --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java
    +++ /dev/null
    @@ -1,41 +0,0 @@
    -/**
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -
    -package org.apache.hadoop.hive;
    -
    -import org.apache.hadoop.hive.conf.HiveConf;
    -import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
    -
    -/**
    - * Test utilities
    - */
    -public class TestUtil {
    - /**
    - * Use this if you want a fresh metastore for your test, without any existing entries.
    - * It updates the configuration to point to new derby dir location
    - * @param conf HiveConf to be updated
    - * @param newloc new location within test temp dir for the metastore db
    - */
    - public static void setNewDerbyDbLocation(HiveConf conf, String newloc) {
    - String newDbLoc = System.getProperty("test.tmp.dir") + newloc + "metastore_db";
    -
    - conf.setVar(ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:;databaseName=" + newDbLoc
    - + ";create=true");
    - }
    -
    -}

    http://git-wip-us.apache.org/repos/asf/hive/blob/3817bb53/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
    index 266d888..0c03a00 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
    @@ -27,7 +27,7 @@ import java.util.Arrays;
      import java.util.Collections;
      import java.util.List;

    -import org.apache.hadoop.hive.TestUtil;
    +import org.apache.hadoop.hive.UtilsForTest;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
      import org.apache.hadoop.hive.ql.CommandNeedRetryException;
    @@ -120,7 +120,7 @@ public class TestHiveAuthorizerShowFilters {
          conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true);
          conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false);
          conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
    - TestUtil.setNewDerbyDbLocation(conf, TestHiveAuthorizerShowFilters.class.getSimpleName());
    + UtilsForTest.setNewDerbyDbLocation(conf, TestHiveAuthorizerShowFilters.class.getSimpleName());

          SessionState.start(conf);
          driver = new Driver(conf);
  • Gates at Jun 5, 2015 at 11:16 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/data/files/vectortab_a_1k
    ----------------------------------------------------------------------
    diff --git a/data/files/vectortab_a_1k b/data/files/vectortab_a_1k
    new file mode 100644
    index 0000000..1b25ac1
    --- /dev/null
    +++ b/data/files/vectortab_a_1k
    @@ -0,0 +1,1000 @@
    +111|||2607|5765.39|3882592.64|3785719054585.463867|true|mathematics|nick xylophone|2065-08-13 19:03:52|2018-11-14 17:26:37.322428829|2086-05-13
    +-87|21091|-1012329052|9182828596851990528|22311.90|2883584.50|3563455992720.427734|true|quiet hour|david quirinius|2047-08-30 02:59:28|2062-07-07 09:32:33.387727593|2083-11-21
    +-58|-31395|-1730740504|302|11322.18|-2081693.61|-1639547871112.334473|true|industrial engineering|xavier laertes|2057-09-02 10:26:29|2074-04-07 03:59:19.110762829|
    +-19|133|95356298|78|-15490.77|-642668.92||true|american history|tom ovid|2059-06-14 15:58:37|2021-07-06 11:56:50.11392413|
    +95|27922|-2027812975|3764|-38980.40|-497550.81|4616197451686.900391|false|education|wendy davidson|2031-12-06 02:13:55|2037-04-01 15:48:02.707720905|2013-05-01
    +23|-9171|-805288503|7432428551399669760|-35391.71|-1387292.76|-4384441919303.938477||nap time|xavier hernandez|2069-02-22 06:47:15|2068-04-02 03:24:11.192596144|2048-06-21
    +7|725||3118|-12519.93|-1305193.84|3984781579512.529297|true|forestry|david davidson|2071-01-16 20:21:17|2049-05-02 13:53:42.230477366|2030-02-20
    +52|8918|-772236518|7250237407877382144|32666.20|-3997512.40|-4361210532142.081055|true|religion||2068-10-07 03:23:30||2010-12-13
    +54|-30304|-146961490|8100036735858401280|38097.34|300734.43|-101907680156.507812|false|wind surfing|alice ellison|2076-07-17 05:35:58|2017-10-19 13:37:33.31074556|1985-06-25
    +36|-10317|-538812082|7792036342592348160|36810.38|1666710.20|1617263763890.299805|true|debate|xavier king||2049-11-23 23:25:42.906731900|2025-10-11
    +-92|21849|868714547|-7496839341561954304|32711.55|-2738225.86|-946920531679.183105|false|biology|zach falkner|2064-05-11 04:54:01|2041-04-22 19:12:27.123637103|2020-04-05
    +-127|7353|1008698636|1719|4983.60|-2636583.28|2290285251062.656250|false|american history|ethan polk|2029-05-28 08:57:08|2071-11-04 02:44:19.546016087|2048-06-29
    +73|16195|737149747|8283099811330506752|5758.00|2121137.53|-3867208889795.392578|true|philosophy|sarah davidson|2061-05-23 23:39:02|2064-12-20 12:52:14.436538366|2053-04-28
    +-46|-28501|810157660|2241|-7769.30|-3860829.99|4232305555838.919922|true|biology|wendy falkner|2054-06-13 14:12:16|2044-03-24 22:32:03.722424719|
    +31|-9609|1541249928|7659279803863146496|-32124.85|1848079.75|-1733147315988.326172|true|philosophy|calvin miller|2058-02-09 03:37:29|2039-10-26 21:00:53.243928394|2030-11-17
    +-35|||1075|21713.21|-4409416.25|-433003793989.392578|false|opthamology||2026-04-19 20:19:23|2060-01-21 11:16:28.358392262|1983-12-12
    +48|7401|476704350|-7911421221625077760|-23364.57|393045.55|-3706576226699.773438|false|zync studies|sarah polk|2064-11-08 08:38:31|2047-09-13 00:05:07.657220269|2073-09-23
    +-29|14773|-1974777102|1774|-41874.29|360339.96|4724948953312.201172|false|undecided|oscar ellison|2058-08-24 22:30:53|2023-10-22 21:22:34.254262972|2064-04-02
    +-57|30921|-522450861|8895174927321243648|-7382.06|-4926003.80|4192159288015.525391|true|topology|david white|2027-03-09 17:30:45|2032-03-22 06:16:46.756458466|2077-08-03
    +|26557|-407089271|661|-38119.53|-51958.20|-4230513318073.718750||quiet hour|wendy carson|2016-08-23 22:18:05|2079-03-05 14:50:42.389412815|2102-03-14
    +-4||273256071|-9014145341570203648|-14145.11|-3116102.10|2704318642566.079102|false||ulysses white|2050-06-25 15:49:09|2013-10-28 19:25:59.70638641|2032-03-19
    +47|23441|-978892011|-7049618574399692800|1466.49|-2506000.67|-1261885101463.384766|false|study skills|xavier underhill|2053-03-08 05:42:09|2059-11-24 17:10:42.836803274|2012-02-18
    +55|-1928|174310705|-6935038507792801792|-4574.16|-2096084.32||false|topology|tom king|2068-11-10 06:35:37|2075-12-17 14:54:53.484250504|2010-11-13
    +-43|2671|851975276|504|35338.95|2409466.90|-4397204039138.387207|false|industrial engineering|xavier xylophone|2044-09-15 08:07:20|2066-11-14 20:08:56.457818450|2097-09-25
    +40|-21772|-1832606512|-8831091081349758976|8799.89|1118606.84|868268330921.963867|false|wind surfing|jessica van buren|2031-08-02 13:03:07|2066-10-02 06:56:23.275227665|2089-06-05
    +-37|20704|-1506324615|923||-4894306.72|3900883250375.564453|false|education|alice ellison|2078-01-19 06:32:53|2034-07-13 13:55:48.524380282|
    +||-419335927|3263|-9973.58|379816.01|-4708070356227.338867|true|religion|sarah van buren|2061-02-13 08:24:35|2074-03-23 07:11:45.666149042|1994-04-22
    +114|-2828|-1144976744|289|-21745.11|1431965.10|-654012413621.529785|true|history|katie robinson|2075-12-28 00:49:24|2028-08-01 10:04:22.203514303|2047-09-09
    +-103|-20934|1107757211|8509508263705477120||507403.73|-4227549209611.147949||zync studies|zach robinson|2062-09-02 06:50:21||2084-08-14
    +-87|||345|-31059.05|-4822074.38|2868093144922.561523|true|education|zach miller|2055-05-09 16:32:47|2015-11-28 12:47:47.540035115|2049-05-06
    +83|-5635|-42151403|281|5719.35|-1928099.64|3951352468527.070312|false|philosophy|calvin young|2016-07-13 04:11:42|2025-05-12 16:31:50.778958822|2036-08-04
    +-51|-20411|929560791|3637|48956.95|-1601399.55|1334219192158.307617|true|values clariffication|david polk|2032-08-10 12:27:00|2059-05-03 02:46:31.520651892|2048-12-21
    +-86|8488|-425196209|-9066993118333706240|28889.32|3917802.65||true|opthamology|sarah nixon|2050-01-29 20:33:45|2052-07-20 02:09:30.619526537|2029-03-12
    +77|-11232|1426152053|7354813692542304256|2007.63|-4307343.30|4686486642663.113281|false|study skills|bob falkner|2018-03-02 05:13:42|2070-06-20 06:01:30.717133325|2042-09-29
    +||546555204|2029|6253.69|2528579.30|-3261089774688.485840|false||victor thompson||2060-02-01 14:16:38.477867192|2081-09-16
    +-44|-16218||-8807361476639629312|36511.51|286248.72|2844066824227.447266|false|religion|xavier van buren|2039-09-01 16:08:42|2065-11-28 01:54:34.188054521|2039-07-18
    +-23|27169|821316302|1371|-21871.99|-3722153.69|-545562668559.717773|false|undecided|mike brown|2063-04-07 13:36:45|2069-02-11 21:42:06.394121082|2054-04-18
    +-75|23220|-1421860505|-7221474017515347968|-11848.57|-4271448.15|-1871570979952.359375|false|mathematics|yuri nixon|2025-10-02 14:38:27|2014-08-18 11:46:05.790786201|2037-05-04
    +23||-677778959|8871707618793996288|-37545.98|2442119.38|-555577054032.596680|false|kindergarten|calvin quirinius|2041-09-15 03:23:20|2017-07-16 08:23:28.728239344|2014-06-03
    +14|-20192|563507584|7198687580227043328|36052.76|-1306614.08|-1346869656422.100586|true|chemistry|katie robinson|2063-06-16 08:30:34|2057-04-22 20:48:38.16105051|2080-07-26
    +-71|30090|1050809633|1990|32100.40|1500446.78|3795668572538.915039|false|wind surfing|tom garcia|2053-07-13 17:41:24|2019-12-19 14:37:16.693138309|2018-01-30
    +-96||-1565671389|8984935029383389184|28720.33|-3180309.64|4280974782610.193359|false|xylophone band|bob underhill|2078-08-06 00:23:13|2034-07-26 23:30:44.175867724|1989-03-16
    +74|23177|1910930064|815|45521.78|-4243565.50|-2179991053030.186035|true|religion|jessica steinbeck|2027-08-11 19:58:53|2025-12-23 12:56:48.785116575|2076-03-16
    +-6|21162|1941527322|2056|18542.60|96721.07|60247412756.062500|true|values clariffication|katie ellison|2027-03-11 08:49:50|2014-05-26 15:11:45.90866024|2099-05-09
    +20|-24115|684561551|8290944180915871744|-32692.85|-793214.39|158268139075.410156|true|philosophy|oscar van buren|2071-11-06 04:15:33|2078-07-06 07:11:57.809667477|2002-07-08
    +121|-18533|167432368|228|-49580.62|-1199504.10|-4410893311038.144531|true|wind surfing|priscilla hernandez|2049-03-14 00:55:26|2015-08-24 11:31:28.973482642|2038-10-27
    +-93|-28769|-463071187||29111.14|3936159.86|855085840126.683594|false|yard duty|quinn young|2074-10-25 23:48:59|2068-09-06 20:05:31.931744370|2092-10-19
    +-95|5736|1204325852|3460|35891.97|3800244.68|-1017339551408.831055|true|philosophy|calvin ovid|2014-10-27 17:15:46|2074-07-15 23:20:20.972096957|1983-04-05
    +-25|-11760|-2069439395||-44985.09|132699.25|1886345168609.331055|true|quiet hour|nick steinbeck|2029-03-23 20:58:01|2031-01-08 05:13:27.828333067|1980-03-09
    +9|-18796|654939016|-8345065519816695808|32691.31|-2340544.58||false|yard duty|katie steinbeck|2056-01-19 20:32:40|2068-12-31 18:53:27.897916012|2095-05-24
    +-115|28393|2080249726|-7576194692683563008|4313.15|311917.45|-2770003968115.233398|false|nap time|ethan robinson|2030-10-10 13:39:56|2044-09-10 23:59:14.562909594|2055-12-02
    +100|6743|1565313938|-8782213262837530624|-964.34|-4407852.66|-773857309922.618652|true|biology|oscar carson|2044-11-17 07:46:19|2053-01-01 14:45:15.764196054|
    +80|-25780|1017953606|7220131672176058368|-35258.17|1908273.76|1088509311003.029297|true|values clariffication|gabriella brown|2075-10-25 05:21:06|2031-10-26 08:37:10.578092307|
    +110|-11008|-2009569943|7452756603516190720|3399.76|1741339.95|-2575772562204.221680|false|education|alice laertes|2058-10-01 22:58:51|2081-02-24 17:48:22.155034035|2032-11-07
    +-38|-15115|-916344293|7410872053689794560|-26858.87|-818719.64|2489707283313.293945|true|wind surfing|nick thompson|2014-06-23 11:31:25|2038-07-05 23:22:59.392777497|2054-03-11
    +0|27116||-7357888618985873408|42003.58|1662438.31|-2435727088707.188965|false|linguistics|mike zipper|2033-07-29 19:41:57|2079-03-13 01:50:52.671710709|
    +|2955|-1463884101|-7409317158045442048|-8186.07|-2572292.78|-2934889488954.759766|false|yard duty|katie ichabod|2015-01-24 13:02:54|2042-06-16 20:51:00.958638266|1994-06-02
    +-30|-12628|868717604|3472|-38487.44|-4162489.36|-4161258810464.367676|true|industrial engineering||2023-12-05 22:32:04|2022-06-14 23:40:08.41754293|2094-06-06
    +59|27852|-1830870295|2494|-24567.97|-929319.13|-3421625759809.039062|true||david quirinius|2074-01-14 13:16:59|2028-01-07 18:05:38.773104432|2002-07-10
    +78|-7850|1677494300|-8992599250893979648|44585.81|-2982899.63|-597466124925.460938|true|forestry|victor van buren|2017-03-29 22:51:02|2068-08-17 17:51:55.371885346|2028-09-02
    +-117|-21922|-799249885|8014986215157530624||2470347.06||true|chemistry|quinn ichabod|2029-05-06 15:39:17|2024-11-10 13:44:54.55064479|1994-03-27
    +48|28558|-78240945|3159|-33993.69|889632.88|3293895628016.016602|false|opthamology|yuri underhill|2020-10-08 08:30:26|2019-11-20 03:42:51.136962161|1988-05-08
    +-76|-3008|-1904737684|||4481669.11|-2023909790198.992188|false|joggying|ethan nixon|2026-11-16 23:20:05|2034-08-07 18:06:43.803487119|1985-04-16
    +112|-6874|203688965|7436133434239229952|7968.70|4867019.14|-1079601478463.523438|true|quiet hour|yuri garcia|2022-06-09 22:16:54|2030-01-13 10:59:10.162327595|2025-08-01
    +-23|-3520|1464695860||41789.11|-3616352.29|3476169263477.153320|false|forestry|alice hernandez|2045-06-14 13:53:38|2027-07-20 19:10:56.885557429|2072-09-25
    +97|-14537|-1358159222|-8845239510002753536|30582.49|772376.99|-2651348117038.914062|false|forestry|quinn king|2016-07-21 22:10:27|2063-04-22 11:22:26.510916644|2044-01-22
    +-88|198|323919214|8345435427356090368|35725.39|530232.57|4000300362554.191406|true|debate|sarah davidson|2052-02-29 16:23:06|2065-11-02 15:21:22.820623860|2016-12-30
    +-19|-32480|2126491387|9094945190752903168|-24257.03|-4364624.36|1971729105497.779297|||oscar underhill|2044-04-04 07:26:25|2033-02-14 16:22:07.768207723|2105-12-20
    +80|-22426|-235819331|-9078662294976061440|-13063.45|-639370.11|-2899905469957.140625|false|study skills|yuri xylophone|2054-03-25 06:38:11|2023-07-21 09:30:19.636575309|2043-03-22
    +33|20884|-909127123|7689489436826804224|-20048.83|-4996960.35|2669260655452.099609|true|geology|rachel xylophone|2076-08-27 00:00:51|2031-07-31 16:48:51.285046883|2037-03-17
    +-20|15688|-1967660827|7620183559667081216|-31608.43|-3778691.05|-253958738593.276367|false|linguistics||2065-07-05 19:36:50|2029-01-01 03:06:13.533276880|2105-01-24
    +-1|-5593|2058640744|-9071565764086521856|-25936.66||-2624029717623.415039|true|forestry|wendy polk|2034-11-26 03:16:39|2073-01-17 06:48:04.722179257|1970-02-07
    +-21|-23540|1678261510|2092|-44267.84|-3266915.53|136230451259.180664|false|yard duty|wendy davidson|2050-05-05 09:31:02|2070-04-03 08:45:42.349508679|1976-07-06
    +-15|-24576|1022707418|1481|46316.04|825604.05|-4275490669201.076172|true|yard duty||2014-06-03 17:24:32|2014-11-29 21:31:04.183698999|2015-04-13
    +-108|-18292|-934008333|8939431770838810624|-19049.65|-3634517.80|308587454891.693359|||ethan ovid|2066-11-25 08:03:39|2046-11-27 13:27:02.11530030|1992-12-03
    +96|-11047|-2096425960|803|4967.48|-2777770.11|4037681204968.351562|false|linguistics|gabriella polk|2019-09-16 21:26:32|2047-10-24 22:32:11.31036942|1984-08-10
    +71|||-8730803262481580032|16660.42|3752718.33|-311540696207.970703|true|history|ethan van buren|2043-08-10 22:21:54|2074-06-06 06:33:09.229039028|1976-08-29
    +-52|-18385|-774406989|8201303040648052736|7836.59|-2532286.37|1913657229799.596680|true|quiet hour|oscar zipper|2039-08-18 17:59:50|2024-02-20 12:49:38.156348041|
    +-11|1847|22308780|-7953426740065312768||-2185323.72|-4536408641834.341797|false|philosophy|mike ovid|2046-04-01 23:23:25|2030-11-15 02:16:54.219567283|1995-07-03
    +-103|-3091|-392713245|-8099313480512716800|40508.97|-2375645.47|-3996919734918.407227|false|chemistry|tom carson|2038-02-20 05:19:48|2057-02-04 09:09:17.378101856|2076-01-14
    +|3694|-1404921781|1972|-31368.73|3279707.14|-2805368749215.979492|true|yard duty|rachel miller|2066-10-28 12:26:53|2065-05-14 17:44:00.934322881|2055-10-15
    +121|-30677|-1061222139|7933040277013962752|-46396.26|958738.91|501053268792.510742|true|topology|xavier garcia|2066-03-24 23:40:52|2047-12-05 05:59:03.709642655|2041-01-13
    +18|-1660|-1955545912|-9189155542884474880|-7714.06|881042.86|2065189794939.748047|false|topology|irene ellison|2026-11-08 06:16:55|2025-05-17 11:03:47.185223340|2074-07-12
    +-42|9028|1621606222|2434|4438.76|-2964984.97|-2918353603075.267578|true|topology|priscilla young|2033-05-06 00:00:14|2023-10-29 17:43:13.522556162|2028-03-26
    +124||978044705|7888238729321496576|-24208.95||-4850732296584.077148|false|undecided|jessica white|2020-12-07 05:16:00|2072-01-10 09:27:24.503516138|2019-05-19
    +56|5599|-665623523|-8240034910581153792|-2335.60|202813.82|3851526875785.361328|true|nap time|sarah johnson|2021-03-07 18:22:05|2049-10-01 06:32:49.357282514|2018-10-27
    +-102|-13722|259524903||-16413.09|-625420.83|3535592833060.938477||yard duty|bob nixon|2066-09-13 11:21:42|2066-04-06 02:06:21.730154464|
    +-56|11238|410340192|3567|29934.62|-2105876.12|2566162905785.595703|false|quiet hour|calvin brown|2018-12-30 00:00:33|2053-12-25 09:43:49.544264273|2095-02-28
    +-111|-797|1412102605|7296164580491075584|31824.08|-886591.25|-2874808713941.199219|false|education|ulysses hernandez|2055-02-28 04:35:16|2032-02-26 21:07:51.12262410|2098-06-15
    +-42|10479|860658464|268|32754.56|-2369331.68|654615772986.453125|true|forestry|fred brown|2024-01-02 15:21:18|2023-03-29 02:26:33.829299976|2062-03-23
    +|30052|-1728171376|-8654433008222797824|35386.65|-1875563.10|-3891050509015.840820|true|forestry|rachel carson|2018-04-15 00:41:38|2016-03-28 02:03:24.513570827|2060-09-17
    +-101|-32455|516843026|2229|-36732.79|-4985474.88||true|nap time|victor ichabod|2051-12-24 02:43:52|2069-12-02 02:14:56.589108032|2030-01-26
    +-75|-15431|877053605|-8783777723063099392|-10499.63|678272.19|3041525159920.972656|false|history|holly falkner|2080-07-05 04:43:29|2034-10-26 21:04:12.771747509|2075-09-17
    +4|8948|1211873318|2897|-17416.18|-4925512.63|-4514053790870.740234|true|opthamology|oscar ellison|2019-04-11 13:20:29|2026-07-07 02:42:24.833351047|2048-12-23
    +-58|1393|1848935036|7989160253372817408|47928.52|-853795.31|-1657076687161.556152|true|forestry|mike van buren|2061-05-06 21:57:53|2058-06-06 16:07:39.838916621|2065-11-16
    +-87|22557|873035819|-8028910243475038208||-2816366.10|-769631198270.062012|true|debate|calvin johnson|2054-12-07 01:08:19|2080-09-25 11:27:16.9717306|2080-12-04
    +83|-10781|-480058682|1286|-17175.04||-2199239017358.216797|true|chemistry|jessica nixon|2040-07-07 10:30:33|2066-09-29 14:31:17.34936376|2096-02-14
    +21|30632|2040926345|8677632093825916928|-45759.70|-2898003.58|-4156272991252.514160|false|geology|yuri underhill|2035-02-11 06:39:18|2017-02-20 17:17:37.184850313|2042-05-26
    +-73|-6948|1773417290|8011181697250631680|11623.01|-958775.93|-1866254854407.820801|false|joggying|oscar brown|2039-02-19 15:50:12|2018-01-08 03:08:57.403292535|2100-01-30
    +60|-19517|-884109192|-7966960765508280320|-10884.65|-3876912.53|-2066003780139.915527|true|yard duty|alice zipper||2062-02-01 03:32:34.383978859|2011-05-28
    +-68|-14315|889733679|9185952983951343616|27500.67|3188754.82|4438755017449.623047|false|philosophy|priscilla zipper|2076-07-13 10:52:27|2016-05-03 07:39:40.516896330|2069-09-26
    +-95|31125|1036391201|7319711402123149312|31964.96|-1757277.11|-1496962050139.103516|true|kindergarten|victor ovid|2032-12-06 07:37:49|2062-07-26 22:38:29.146178601|2032-04-09
    +-94|-6088|1604076720|9119046173224370176|19740.52|-4678968.17|3557302643795.097656|true|american history|david xylophone|2073-01-23 03:17:41|2040-11-18 04:31:12.709589656|2004-09-28
    +19|-9243|-1249011023|-8760655406971863040|-32655.55|-1649087.83|932399155849.453125|false|forestry|alice van buren|2018-07-11 06:15:11|2027-04-13 10:06:40.902651522|2019-12-25
    +12|7343|1418228573|8190539859890601984|-20276.81|-4235586.20|-1196769263706.656250|true|values clariffication||2070-05-15 09:16:30|2078-03-22 13:36:17.940377243|2037-02-13
    +-12|6813|-253084551|3974|40698.97|-1115432.48|-2448300846308.981934|false|wind surfing|wendy ichabod|2022-05-20 11:14:25|2053-10-25 16:58:08.223136198|2097-11-18
    +-125|-22641|1597303154|2106|4874.58||-30490934435.612305|false|religion||2015-03-28 09:25:39|2013-03-04 23:58:15.71378943|2078-11-03
    +39|8410|1640445482|2579|-14853.40|-3366426.92|1755470105685.397461|false|mathematics|jessica xylophone|||2081-02-20
    +|-12755|-1353470095|7210160489915236352||-1899846.57|-653238714138.346680|true|undecided|priscilla xylophone|2022-03-08 18:50:03|2059-09-27 11:04:07.90814681|2011-06-29
    +102|28247|794783516|-7907355742053883904|20683.15|2965661.82|-624867269213.389648|false|chemistry|mike van buren|2064-01-12 01:22:43|2055-03-27 10:02:55.174795513|2085-03-01
    +114|19513|418182899|1846|-24886.47|-3030589.60|4183756295927.345703|true|education|luke xylophone|2017-10-15 17:18:43|2063-06-10 14:18:48.104797455|2044-04-24
    +108|27636|215508794|1189|-25835.00|4453603.41|4405275049808.744141|true|nap time|ulysses van buren|2015-07-02 03:45:28|2013-12-31 08:25:18.118248757|1981-11-06
    +23|-19623|-1039040287||-6512.80|-660440.82|-1642262021182.041504|false|geology|xavier quirinius|2080-12-09 03:55:10||1998-04-16
    +-53|-4169|879290165|68|34884.11|-1267493.50|-3279094975952.290039|false|industrial engineering|mike young|2075-12-04 18:22:09|2054-10-01 19:12:15.766024626|1977-12-27
    +-88|16796|1151752586|-6975459232300236800|-4784.15|-4864373.89|3221553706534.408203|true|topology|tom zipper|2064-04-09 08:01:25|2070-08-17 06:01:00.968011813|2027-04-10
    +13|17531|1805308672|-7700203302632210432|-37763.56|3704107.83|3425256239457.610352|false|undecided|xavier thompson|2027-04-18 16:26:01|2024-07-18 19:36:10.425538948|2092-04-17
    +-61|32019|1420099773|-8021859935185928192|38974.63|1673755.73|2102524829645.560547|true|chemistry|ulysses falkner|2029-02-04 06:51:54|2046-05-19 15:05:25.732127788|2062-03-10
    +7|19302|538268118|7339426767877390336|20672.29|4555902.70||false|topology|fred laertes|2017-01-13 01:23:09|2016-05-16 23:25:47.63891547|1998-12-23
    +-114|-28736|449788961|774|-18622.98|-1268938.21|2666440101920.493164|false||priscilla ichabod|2063-12-27 01:29:50|2035-03-18 21:03:10.648253783|2062-10-07
    +101||-1914072976|2295|1339.82|4112381.85|-1984449836425.916504|false|study skills|alice ovid||2075-09-26 04:56:48.660923039|2050-07-10
    +44|-18090|259204652|-8559252110266564608|-9038.73|-4718571.75|3963517992293.716797|true|forestry|victor white|2066-12-24 15:59:32|2038-03-27 02:23:06.834673196|2096-07-12
    +-24|10161|-882028850|3079|-45314.95|1053922.25|1574414367849.214844|false|history|xavier ovid|2052-09-14 19:24:26|2034-04-25 18:26:08.505694709|2040-08-03
    +89|-24801|-158233823|1346|42990.33|4421723.61|2978558145914.504883|true|geology||2028-06-15 20:46:41|2071-05-17 17:16:54.148951459|1989-11-05
    +31|-31582|1219616145|-8007017894942638080|-44727.28|-4114788.64|-4378058802654.081055|true|american history|quinn quirinius|2060-09-20 08:49:06|2023-11-29 06:13:18.611946024|2019-12-11
    +|-19159|1807877618|987|-5972.19|-119465.75|1452475353908.404297|false|quiet hour|holly ichabod|2062-09-02 18:18:39|2070-04-17 08:28:26.5503386|2075-09-26
    +27|17023|1747664003|-9049720998034137088|22254.70|3306808.08|3177841668890.231445|false|biology|priscilla ellison|2015-07-10 05:27:46|2027-06-11 15:58:30.658173981|2024-08-23
    +|-25344||-8158011642485825536|-48220.79|4820070.80|3955085450779.839844|true|chemistry|victor polk|2055-12-20 16:02:07|2024-09-03 18:32:19.912271231|2011-06-12
    +-96|-6950|-1070951602|7697541332524376064|-23329.71|4539092.16|-1684983372492.716797|false|undecided||2069-02-08 07:45:14|2068-03-26 18:14:15.165533561|
    +-74|4261|217823040|8773222500321361920|-11741.38|1094988.55|3709069978104.153320|true|topology|rachel xylophone|2044-11-15 10:42:51|2059-08-26 20:14:15.307180062|2022-10-12
    +-72|-7178|1814570016|-7915999634274369536|30236.61|2437780.95|2666538537304.746094|false|biology|fred carson||2079-01-28 01:30:38.861840968|2077-05-11
    +-64|-1725|1493555718|6974475559697768448|-16776.52|-4105739.18|-2442617490401.950195|true|education|katie thompson|2044-03-29 00:20:37|2015-04-18 09:51:54.990681181|2078-01-13
    +41|-19270|1119976718|7052226236896256000|28023.88|2110273.07|-4751768490466.296875|false|biology|yuri laertes|2079-09-26 20:39:48|2068-11-17 19:45:06.599632615|1983-05-31
    +-58|-6494|1222217404|650|37717.83|107717.64|3643292384671.887695|true|industrial engineering|luke steinbeck|2028-02-25 14:55:40|2074-07-02 03:20:40.417209896|1973-03-04
    +-127|-4252||3079|39550.67|-3279977.09|18225457799.800781|false|zync studies|holly zipper|2045-08-14 20:11:34|2024-05-11 15:11:48.691514072|2074-05-09
    +-26|10261|914062370|1039||2283246.90|-1345490060709.941895|false|religion|luke robinson|2039-04-22 14:44:00|2053-03-18 12:47:22.448344815|1979-11-27
    +-46|7356|683320224|-7610137349734883328|23741.83|-4065786.49|4723266191273.687500|true|chemistry|nick carson|2022-07-31 17:56:09|2024-03-03 22:59:52.779904666|2022-08-11
    +48|28309|-570632618|-7526793959592140800|-4768.36|-4329360.25|3073439981743.923828|false|joggying|victor laertes|2034-03-18 16:11:32||2086-05-21
    +39|3228|-535056977|8897901899039473664|3865.37|-206033.60|3887729126314.027344||study skills|wendy laertes|2018-03-11 05:09:22|2034-09-09 08:05:25.434342520|1982-04-12
    +-73|-1011|-1058166020|8793387410919038976|-9572.10|2650235.31|-3040241790788.478516|false|quiet hour|oscar robinson|2062-03-17 08:07:25|2042-08-08 14:24:00.510838590|2078-05-19
    +-65|17073|1111985530|9112400579327483904|46710.33|130023.02|||industrial engineering|wendy falkner|2051-09-14 21:51:35||1994-06-09
    +-127|29365|895945459|-8559008501282832384|19094.69||-4601795255939.329102|true|study skills|katie underhill|2062-08-08 00:34:44|2035-03-17 18:33:10.848870183|2087-04-30
    +42||-997463353|1788|-7.25|-1624411.04|2061492866787.446289|false|debate|xavier white|2074-08-12 02:19:10|2024-07-12 04:37:59.670091691|2079-05-31
    +-24|-13978|-491882534|8554899472487596032|-42196.85|2380170.34|995819157271.000977|true|debate|ethan garcia|2067-04-29 08:24:05|2033-02-23 23:45:43.224368846|2026-11-24
    +45|-7865|829101712|-8070535484085895168|-8695.03|1393839.55|-2930125300660.009277|true|religion|calvin garcia|2069-10-10 07:46:10|2030-09-03 02:47:09.999512465|2094-08-04
    +34|-32240||-7858505678035951616|21913.94|-4848416.07|-4417461834328.939453|true|yard duty|tom miller|2020-08-15 02:15:04|2049-01-07 01:23:16.815364884|2041-04-17
    +82|-4550|-1497098905|1058|19349.22|-3207956.11|2622882069011.147461|false|education|gabriella nixon|2048-06-12 05:15:59|2070-06-08 15:10:15.103401622|2047-07-22
    +90|12762|-983874694|-7598782894648565760|-6545.09|4223917.78|-2724893143053.712891|true|biology|wendy carson|2064-09-28 23:47:20|2057-05-10 17:22:29.44470118|2009-10-04
    +-52|-15501|-1184620079|7947544013461512192|17444.50|-4019474.98|-4273574634879.970703|true|debate|wendy falkner|2033-04-24 23:46:39|2064-12-09 12:41:18.605098238|1999-04-04
    +-121|-19282|787925706||-39608.35|-3490480.46|-1396998641989.632812|true|opthamology|xavier robinson|2050-05-29 18:50:17|2061-07-15 16:24:02.41944450|2064-10-08
    +-104|-6736|272086526|8625937019655200768|-42455.39|-2155542.09|-3341536888638.039062|true|undecided|wendy xylophone|2052-10-28 04:50:15||2103-12-05
    +-105|-4059|760466914|3701|12507.09|2661135.97|1085001013184.765625|true|history||2056-10-06 10:43:05|2024-10-19 16:47:43.958389682|1991-09-28
    +90|-25683|745725681|-7692192232238678016|364.62|-695924.61|-918004138590.973145|false|yard duty|mike polk|2052-05-18 02:04:21||2015-02-25
    +-90|9110|1398486099|1291|-18652.98|4330260.30|-2988564000800.586426|false|chemistry|sarah robinson|2030-12-19 06:51:08|2073-06-18 12:33:18.634781669|1999-02-06
    +53|24715|476919973|3887|-33746.36|-1812815.81|2248167382355.781250||values clariffication|calvin garcia|2028-06-28 08:30:31|2039-03-21 22:46:13.628314022|1994-05-06
    +-112|14286|287239980|1462|23269.50|-3421045.14|-4825619680711.810547|false|geology|katie johnson|2023-07-03 15:05:18|2018-11-09 18:46:47.759800359|1983-09-25
    +-36|3213|-1528033060|-8485389240529354752|33155.26|3831347.85|-2604012470736.947266|false|study skills|ulysses ichabod|2039-07-17 08:06:17|2032-05-11 08:35:57.629571826|2033-04-05
    +-122|-16872|1520375588|8660248367767076864|-46409.01|1435619.95|-301617019681.543945|true|nap time|david steinbeck|2028-09-21 14:32:03|2053-05-02 22:50:04.195045396|1977-04-22
    +|32092|-951728053|-8115963579415650304|17565.73|-4855596.00|-1567705616965.078125|false|wind surfing|ethan ovid|2079-03-11 06:37:16|2080-10-21 08:32:40.577382791|2021-08-16
    +108|-6933|-66112513|388|-23820.11|-2660889.99|4673408308800.226562|true|values clariffication|calvin brown|2013-07-21 19:03:27||2078-03-23
    +79|-6142|1295073553|-7262049693594943488|34920.29|-2775037.93|-3850646756153.040039||chemistry|sarah king|2062-11-28 06:34:11|2064-12-02 14:20:25.689540381|1970-10-19
    +114|17164|-217785690|1343|-22425.89|-4109302.98|1542043552284.902344|true|biology|mike miller|2043-01-06 00:29:33|2027-09-11 18:50:09.224829213|2084-05-01
    +79|22366|1632769786|1508|-16193.28|-4581299.04|-4048240005182.783691|true|opthamology|sarah zipper|2027-06-15 04:28:56|2024-11-18 13:11:30.807086190|2094-07-09
    +-65|5785|-648766606|1701|-25497.04|3706795.80|4406779957872.437500|true|study skills|tom polk||2047-01-20 19:02:44.296212906|2079-09-24
    +-23|-29475|2052773366|8837420822750314496|-44389.81|-3244548.09|-3320189294654.056641|true|xylophone band|katie carson|2014-08-14 04:46:15|2032-01-06 02:52:13.841451923|2008-01-10
    +25|24014|-1318045616|-8704234107608203264|-25094.94||1313758909596.565430|false|study skills|irene young|2034-08-06 10:56:24|2050-11-22 00:49:32.763404963|
    +67|-9614|1124269631|3680|29490.38|-1942538.00|-2214662106936.273926|false|topology|katie ovid||2028-06-06 03:25:20.277859556|2088-10-27
    +23|-21708|-94709066|7271887863395459072|-43725.17|-582351.40|-1251943808107.505371|true|mathematics|oscar garcia||2019-08-07 09:24:28.969847804|2091-10-12
    +-49|-12071|1384071499|8156018594610790400|44500.74|-423299.81||false|mathematics|zach garcia|2062-01-17 23:46:46|2035-06-22 17:50:21.193860499|2068-12-27
    +-34|-27553|1860113703|8708845895460577280||4443055.22|-3152882360663.084961|false|geology|priscilla young|2032-12-02 17:45:28|2030-02-07 12:05:50.442818058|1981-07-26
    +||1634441052|2847|-30620.44|-4164391.70|4767856866280.625000|true|kindergarten|rachel steinbeck||2015-02-23 17:38:53.735735894|2038-02-01
    +-126|-31537|693331761|2509|-25833.15|-4801160.87|2804616317735.586914|false|kindergarten|priscilla thompson|2046-02-05 15:38:03|2035-04-25 16:30:39.959832372|1983-09-26
    +121|-6219|65956045|1987|-13776.59||3855241018435.791016|false|joggying|nick laertes|2076-07-22 18:55:23|2045-05-01 12:49:24.875445|2104-07-12
    +-8|30861||8470141334513098752|6021.11|3057994.15||true|undecided|gabriella thompson|2056-03-17 16:36:40|2076-09-17 19:51:03.971998194|2019-06-03
    +108|-10687|977292235|2108|34147.75|3835815.78|-3242583557065.046387|true|xylophone band|luke van buren||2026-07-29 12:13:07.350517258|2004-10-09
    +-103|5235|283322761|-7623405558242500608|-9121.56|4969099.99|1418203690420.333984|true|history|ethan ichabod|2052-01-27 14:18:09|2076-09-25 20:35:14.535725735|1995-08-27
    +-61|3063|-870624802|8687042963221159936|3943.90|-209028.71|-2183718197982.749023|false|joggying|calvin davidson|2034-06-08 16:02:13|2029-09-24 14:25:09.37764381|2105-01-30
    +-96|-10516||3568|-19009.76|3239989.12|-4784778822088.213867|true|topology|oscar nixon||2036-02-03 04:58:16.203066567|2065-12-22
    +-50|-30748|-1735287250|7746402369011277824|-4179.91|1292492.11|2324680845107.813477|false|mathematics|david allen|2016-06-22 21:51:17|2069-09-12 15:00:57.362462590|2072-09-24
    +-103|9969|-1703620970|-8387536830476820480||-3049943.90|4709582975677.548828|false|zync studies|calvin allen||2079-11-27 18:43:43.659151983|2044-09-10
    +34|-21818|818313200|3060|-38828.51|-2063369.12|-3583867768957.524902|false|debate|sarah johnson|2064-02-16 11:03:07|2019-10-12 04:27:17.822098134|2004-10-26
    +-85|19350|1409872356|-7512289590991544320|-10184.90|3793945.90|-923624349228.369141|false|industrial engineering|katie white|2032-02-14 12:11:48|2061-04-03 12:08:01.5162482|2081-03-17
    +-78|-28932|187893585|8235179243092090880|-23609.39|962981.03|2966147671023.726562|false|zync studies|luke ichabod|2017-01-05 14:49:04|2059-12-23 13:06:00.979846594|1993-11-14
    +92|2728|398960205|8540237852367446016|-8750.79|-4703249.19|-501603049952.486328|false|nap time|tom king|2030-04-28 01:59:00|2018-08-22 06:45:33.15399240|1977-02-18
    +8|6144|1437057145|-7265998318110711808|-21666.49|361449.07|-3148061620108.357422|true|quiet hour|luke johnson|2065-11-30 23:06:00|2052-07-18 07:53:41.806082172|2062-06-04
    +-120|26859|-1554325042|583|15171.32|-2158930.81|-301298241024.774414|false|undecided|victor van buren|2050-07-22 13:12:51|2063-08-25 00:41:53.424629732|1979-04-17
    +-124|-10095|1033609549|-8318886086186213376||3677625.78|4672389589988.128906|false|history|irene johnson|2056-03-14 14:13:51|2077-11-14 16:16:46.603310412|2084-07-26
    +75|-23638||2410||-968637.64|1698116680328.283203|true|debate|rachel allen|2042-10-06 19:18:58|2057-08-09 04:27:05.138333574|1993-11-01
    +-59|1361|-360113158|3021|10379.72|-2330239.84|2488834645794.498047|false|opthamology|bob king|2063-06-09 19:51:43|2037-08-18 09:27:00.913006136|2011-10-15
    +|3523|661380540|-8018511948141748224|31005.68|-1939975.71|561437334048.342773|true|wind surfing|quinn thompson||1970-01-01 00:00:00.212340798|2086-01-31
    +40|-32296|84231802|2886|-4154.25|4389868.65|378820126909.822266|false|american history|holly laertes||2065-08-04 06:59:17.824387481|2014-11-15
    +-32|-6637|-1726479726|-7831320202242228224|39531.99|2377538.79|1099330213298.444336|false|debate|victor davidson|2050-04-05 23:38:10|2022-03-12 21:56:07.234951116|2018-10-08
    +84|-18387|346562088|7998357471114969088|44620.73|4045062.26|4748679788424.253906|true|kindergarten|zach quirinius|2020-07-07 16:26:44|2047-03-31 09:15:12.887634795|2084-02-05
    +-27|-10815|1769324649|7936149988210212864|11543.56|227674.23|-812606022147.335449|true|zync studies|alice johnson|2048-05-19 07:13:08|2062-03-12 06:53:03.331725965|2076-03-18
    +81|2643|2140632003|-6970396058557005824|19854.94|2938603.02|562510896793.071289|true|education|david davidson|2068-04-12 18:18:24|2041-04-04 08:06:20.143031301|1972-02-24
    +-24||780938234|-7083646746411720704|-16507.75|-3851733.33|1863907193854.252930|true|joggying|luke falkner|2049-02-04 12:28:05|2057-01-09 07:22:38.778950773|2100-08-23
    +-101|-8857|-1918651448|-8675661101615489024|43481.62|149175.06|-2178760699903.925293|false|joggying|mike davidson|2058-06-04 16:19:32|2072-08-14 09:56:17.194554282|1976-03-18
    +|-10821|770574055|-9126793997498957824|-36806.07|94051.69|3234847750887.297852|true|yard duty||2051-04-08 16:12:47|2075-11-21 21:55:19.18153321|2066-08-28
    +42|9158|776606164|2506|-24639.80|-4869417.64|3079389364956.313477|true|industrial engineering|alice laertes|2044-10-22 22:17:52|2074-08-21 15:27:43.96892721|2058-05-29
    +|7180|-51612681|7249443195032985600|-7568.74|1202707.37|-2119064600783.993164|false|undecided|calvin ichabod||2022-04-25 10:39:59.377128631|1982-07-06
    +29|10891|1153811197|-8243487285852766208|11569.34|4212709.34|1034657320312.746094|false|american history|victor brown|2031-09-30 15:01:06|2025-09-04 14:13:00.976690553|1993-05-12
    +-122|7654|-734921821|3941|7256.71|517028.01|-1114345601613.280273|false|topology|zach carson||2070-05-25 06:36:39.297148492|2001-05-07
    +-50|-28968|-1460613213|8875745082589929472|-7442.32|-2702250.82||false|philosophy|priscilla brown|2032-06-20 09:31:24|2052-09-03 16:21:52.208215796|2087-08-18
    +-112|-29907|206121314|-8705403811649355776|-44201.12|-897624.69|-2173857521374.103027|true|xylophone band|xavier brown|2059-05-21 10:37:20|2065-01-02 03:32:19.285960165|1998-03-18
    +-6|17701|-1606567895|8410599906334097408|23194.74|4572852.32|-1064163809823.100098|true|xylophone band|holly nixon|2044-04-30 10:00:19||2011-12-07
    +-117|-5869|1523657918|-8400045653258444800|-37316.59|-1140903.56|3668214733218.100586||study skills|fred brown|2066-10-11 19:46:58|2057-04-21 18:12:07.438157094|2072-06-21
    +-58|-2118|-37876543|-8859107121649893376|-2682.16|-2318215.28|4457864918964.134766|false|zync studies|bob van buren|2065-12-18 19:50:19|2043-05-16 22:15:58.560013699|2000-07-15
    +19|-26060|1425362689|2201|35741.32|-635373.52|-1660120664738.187012|false||mike quirinius|2055-11-26 10:21:07|2043-11-02 23:23:13.175695849|1978-08-05
    +-115|1755|596045726|-7000925438663041024|1169.13|1224590.95|4349386575138.148438|true|xylophone band|calvin quirinius|2014-03-11 02:26:36|2019-09-03 01:49:36.181933289|2088-02-28
    +-95|-2617|1229172951|490|-18461.73|1170976.95|1480054571327.649414|false|mathematics|quinn hernandez|2021-05-20 06:00:47|2014-10-28 09:16:31.610844395|2080-12-12
    +30|-28244|1978200605|1910|7799.30|-123266.43|-1103541868445.567871|false|undecided|quinn underhill|2019-08-08 22:46:59|2059-05-28 04:51:35.498984116|1981-10-31
    +-83|-15497|1000106109|8900351886974279680|46362.81|-4358428.80|3426770165622.876953|false||ethan brown|2027-01-01 03:13:01|2063-08-29 02:30:48.169621762|1992-08-21
    +-93|4032|879289168|7343171468838567936|29799.26|-3707213.21|-3700102015935.797852||education||2080-10-01 22:58:13|2042-10-22 03:29:04.44238849|2038-02-21
    +7|-29895|-464804906|2131|-21839.05|4389713.57|-1713151304480.397949|true|quiet hour|luke ellison|2027-10-30 14:21:42||2066-10-25
    +-106|-22071|-1247229632||-38427.52|-2639190.56|-755064468729.002930|true|religion|victor robinson|2044-02-05 12:48:23|2058-01-26 21:26:48.253653944|2010-09-07
    +-72|-5907|-211669740|3208|-38197.98|781541.24|-2726366048183.592773|false|industrial engineering|alice allen|2036-11-15 10:48:44|2074-12-03 23:58:03.654571719|2076-11-09
    +-8|10600|-1155174991|-7420448501073051648|-48598.14|-2840797.35|966723065681.023438||kindergarten|luke robinson|2075-10-15 20:32:00||1990-02-19
    +-89|-8267|786565385|3418|40746.89|-3336502.60|-3655334890183.326172|false|history|quinn van buren|2073-12-08 10:59:27|2059-12-18 19:47:35.135760242|1992-09-21
    +58|24847|1115197541|-7030489936116252672|-32967.44|-3216068.70|-4540163781577.817383|true|study skills|ethan quirinius|2057-07-19 17:20:26|2061-01-07 10:20:58.287466414|2058-04-02
    +82|||1093|-41542.22|-2852753.19||false|mathematics|oscar ellison|2078-03-12 21:36:59||2040-03-02
    +7|17107|-1227085134|454|-45679.81|4280344.76|-4755957353103.070312|true|biology|victor robinson|2075-06-02 02:53:30|2040-12-08 01:16:10.194735899|2020-12-19
    +-109|-25692|-714594143|2791|15638.71|4868354.39|-1920412941995.195312|true|yard duty|priscilla brown|2013-03-25 15:38:28|2039-10-25 16:44:04.385364994|2073-11-05
    +98|19215|30036142|1252|5512.48|2861312.90|304220501137.706055|true|zync studies|wendy polk|2074-09-18 04:34:04|2042-10-10 04:18:49.518287517|1992-09-29
    +-119|-20879|-1900894010|19|-14328.52|3163463.29|-2231521147965.978027||american history|xavier van buren|2051-08-02 17:01:13|2070-06-10 09:08:31.433429764|2049-12-19
    +-80|11979|874824958|485|-49403.10|1836049.78|-3891815088473.258301|false|american history|ethan king|2023-03-21 16:09:44|2018-10-09 10:15:27.268042756|
    +-122|4568|1505665168|-7453525026342617088|-47688.82||-2370722163282.117188|false|mathematics||2043-02-15 19:11:53|2080-04-06 03:34:04.633286901|2079-05-24
    +|2677|-2077771325|1604|4199.81|1851095.86|-3091236716436.114258|true|philosophy|fred brown|2051-05-22 03:56:26|2073-04-07 19:52:50.601742131|2023-01-10
    +62|-14280|-2043805661|8365058996333953024|11341.10|4541245.28|4402724332930.460938|true|xylophone band|yuri robinson|2080-04-12 01:21:14|2031-04-06 19:14:59.55187421|2031-05-31
    +62|13578|738356485||22454.83|-4779.61|-1857979799639.935059|false|nap time|ethan king|2046-10-08 18:53:18|2058-12-03 23:22:35.944838503|1971-07-29
    +-48|-22938|340384179|-7877598807023386624|2517.88|-2373948.65|947585735001.510742|false|opthamology|david thompson|2075-07-24 01:40:11|2036-06-14 00:00:15.278560351|2003-09-18
    +-108|-23194|-1198465530|3055|-36853.20|2315107.36|-1344917461892.407715|true||rachel quirinius|2041-04-15 03:29:16|2043-07-29 21:24:09.491072788|
    +-112|19350|-705887590|-9079801920509001728|42578.38|2583540.14|885849762897.948242|false|american history|ethan white|2055-05-21 11:56:02|2059-12-29 12:16:34.533626678|1984-03-14
    +33|16430|1347876055|-8465978403747037184||2398238.05|3976258329685.542969|false|undecided|david quirinius|2040-12-15 14:20:04|2059-03-26 08:14:53.410162567|2013-04-21
    +|-21574|62293025|2426|14978.09|3616509.09|3825560324453.007812|true|history|sarah falkner|2035-04-19 13:32:57|2061-05-10 03:23:57.557730657|2099-03-03
    +5|-6114|198017473|135|-14085.31|2000803.23|-2919243579540.449219|true|american history|nick brown|2073-05-02 15:11:49|2047-05-06 13:18:00.344896111|1997-06-17
    +35|28128|1286367391|-7488415863027367936|-26817.08|-797714.37|3875352447190.109375|true|study skills|jessica brown|2072-03-19 14:27:46|2018-12-18 23:20:30.63266134|2080-12-24
    +78|-22358|376076075|-9136398397785948160|46846.67|-1700725.96|-899635084786.325684|false|industrial engineering|katie king|2017-10-06 08:33:37|2057-10-12 20:55:36.383445468|1972-02-28
    +-34|3009|-1804244259|579||-2740315.70|362659766665.806641|true|zync studies|sarah thompson|2032-07-12 22:13:10|2035-11-06 17:59:27.77357393|1996-08-02
    +-103|4081|2083836439|8144552446127972352|36570.50|3555782.96|3609640367332.538086|false|philosophy|sarah davidson|2031-08-26 09:58:56|2023-01-11 17:04:23.449037266|2041-06-07
    +39|-16367|1776456512|6933451028794925056|-43403.36|-4299638.88|793504824242.568359|false|forestry|ulysses thompson|2028-12-06 04:06:25|2055-06-24 20:19:48.682407175|1971-01-13
    +37|20128||1777|35965.37|-1496116.32|-3499447525525.525391|false|linguistics|calvin robinson|2067-08-21 11:00:04|2033-11-12 22:25:15.735331075|2019-10-13
    +45|20254|1033836308|-8927968289860370432|-47362.97|3785304.85|-3394951194042.761230|false|forestry|david thompson|2017-04-09 00:02:35|2054-01-29 09:25:44.815093672|
    +-99|26910|-571587579|7265141874315517952|-28011.29|-4797236.03|4313736848665.250000|false|wind surfing|katie underhill|2080-01-07 15:47:45|2057-04-03 15:14:56.240582613|2046-03-03
    +-118|16437|727802564|4078|32210.62|913010.81|2387469547668.992188|false|undecided|fred ellison|2030-03-31 06:22:45|2061-03-01 05:58:51.835198480|1982-08-19
    +98|-11929|596242714|443|-18850.34|-2514812.80|3841757426531.521484|false|undecided|tom miller|2056-04-06 22:21:49|2023-11-11 06:58:15.825259324|2070-01-08
    +-1|-22217|-743680989|7391208370547269632|1450.95|3168984.07|422951017333.055664|false|topology|priscilla thompson|2047-05-23 07:19:55|2019-08-24 23:35:52.785182782|2092-08-23
    +-23|32589|1575091509|7790728456522784768|43789.16|-3767707.87|449819577744.783203|true|forestry|quinn thompson|2049-08-29 23:03:28|2059-03-17 09:44:04.511510153|2082-09-14
    +-48|-22447|669871113|8398862954249560064|45334.49|-168095.10|-95855026425.759766|true|quiet hour|ethan laertes|2026-06-26 02:42:50|2014-08-21 12:14:15.695317873|2075-09-12
    +114|27046|397255100|-8836899523028312064|30215.93|-717006.21|-357328163291.050781|true||zach brown|2073-12-21 08:21:58|2061-05-01 00:50:27.746635942|2021-06-09
    +9|-11679|-2076886223|-7805985795815342080|19575.50|120994.39|380373641521.103516|true|xylophone band|ethan carson|2076-07-30 01:09:00|2057-09-20 07:35:34.305010925|2065-08-02
    +-5|14783|531459992|1439|19351.91|4803315.07|-574114492621.049805|true|topology|jessica thompson|2072-08-04 11:14:51|2025-01-30 01:47:54.411372161|2085-08-12
    +-41|22870|177391521|7266437490436341760|12.17|4688982.65|-4069350543654.266602|false|religion|oscar falkner|2016-04-04 12:22:03|2056-09-18 10:09:51.803143104|2104-08-29
    +-123|7959|311478497|-8852770376039219200|-44161.63|1078252.98|1685122154602.396484|true|values clariffication|irene hernandez|2022-06-21 13:36:54|2057-10-24 03:22:01.465719271|2102-04-28
    +82|-17995|673904922|3119|31236.39|3313603.14|350230667228.643555|true|religion|calvin steinbeck|2035-06-03 08:40:07|2047-07-13 00:30:59.312757955|1979-08-05
    +109|-28299|1042237722|2962|-49297.70|2544504.56|1300896197981.842773|false|kindergarten|gabriella falkner|2045-09-01 02:14:09|2049-12-12 02:33:51.210858024|2041-12-08
    +|7483|-1974257754|9001907486943993856|-39292.53|2604166.52|2628816035521.843750|false|mathematics|jessica nixon||2024-04-14 05:51:51.799661240|1973-10-11
    +37|29639|218917585|-7378096180613840896|2468.85|545061.80|4576650304788.998047|false|joggying|quinn ellison|2030-03-18 10:56:59|2064-04-10 13:26:19.857498352|1973-05-29
    +54|13145||7091300332052062208|-17351.53|4747854.89|3817727970940.189453|true|topology|oscar thompson|2065-09-11 19:52:17|2038-02-26 14:34:48.662632111|2098-03-08
    +79|-4491|-559270035|-7818454479651135488|32811.79|-2589530.66|3650613746347.026367|false|forestry|zach polk|2062-04-28 22:14:44|2053-08-03 04:16:15.344431898|
    +123|31427|-1131684944|-8431492599012163584|48223.45|-4797838.11|4766118630785.527344|false|philosophy|quinn white|2035-12-20 20:20:18|2055-03-06 04:00:57.170922096|2071-06-23
    +-57|-3602|-373034494|-8521578237232529408|35310.58|523737.60|4138846175212.621094||industrial engineering|ethan polk||2070-05-20 09:18:23.921779415|2089-12-19
    +-6|-16722|-490337498|1926|43246.64|-22165.47|-624395488497.526855|false|nap time|jessica falkner|2070-09-14 21:01:42|2024-10-17 22:24:01.153750276|2069-10-18
    +64|-3219|-1829691116|7490717730239250432|46586.47|-1915228.77|1287655745842.997070|true|wind surfing|fred ellison|2060-10-10 10:17:38|2080-01-01 16:19:34.640061653|2095-02-04
    +-26|-20949||-9187662685618348032|43398.02|-49310.50|1863250559721.372070|true|study skills|katie carson|2048-04-09 07:40:02|2073-09-18 13:35:45.258178237|1984-12-02
    +-50|13078|135341845|2016|47991.75||-3445060192851.059570|false|zync studies|rachel young|2024-12-31 09:01:02||1970-08-12
    +-41|27758|1646811064|1153|-4313.26|-3049537.40|-69377778943.611328|true|topology|katie johnson|2041-02-02 14:22:59|2041-06-19 10:09:46.112258643|2041-07-24
    +-70|26232|-1447263708|2277|41062.58|-2943975.09|-2631454750987.565430|false|linguistics|yuri miller|2051-01-04 12:34:39|2075-09-08 01:57:00.107926096|2070-02-10
    +|-22858|1743696703|3300|-4380.97|-2069211.91|-4596153868922.030273|false|study skills|rachel van buren|2080-08-26 15:15:51|2030-04-01 16:16:55.346669591|2015-09-27
    +-40|3354|1303632852|3244|-21821.23|3976574.38|-4697749109705.853516|false|zync studies|quinn robinson|2071-08-03 03:03:18|2033-11-19 04:14:03.372021127|2093-10-02
    +-56|11120|1949494660|7107604675626008576|11034.18|3802098.40|-3918244512545.996094|false|wind surfing|calvin nixon|2022-03-17 12:57:17|2020-11-30 18:42:55.661360529|2038-10-05
    +-57|-20329|-1857500489|1198|13491.58|-2693679.28|755404222558.269531|true|mathematics|oscar ellison||2033-01-29 05:36:21.574403565|2092-09-14
    +|-6735|-2138343289|3401|-22767.90|2032418.93|2618674044147.155273|false|xylophone band|xavier davidson|2019-01-23 07:48:23|2018-12-23 15:42:16.611413141|2046-12-15
    +-22|10761|-1226425562|7718825401976684544|8162.40|1047714.74|-2476610981011.973145|false|opthamology|mike robinson|2057-11-26 17:53:27|2069-08-23 05:01:36.124435431|2092-08-24
    +-40|29502|-809805200|-8518258741831680000||1144686.92|-4590760977006.702148|true|mathematics||2036-10-09 18:06:29|2064-10-14 13:41:49.171097321|2081-04-05
    +50|24081|-728541537||48961.12|-4012888.15|1070384818656.826172|true|topology|victor xylophone|2016-05-04 04:35:48|2038-11-09 14:41:33.447548472|2051-02-08
    +102|-6073|1464703053|2004|-38318.30|3968580.02||false|yard duty|david white|2046-02-20 04:57:04|2060-04-05 20:06:25.19768150|2102-06-28
    +-41|-11863|-116484575|7801697837312884736|38211.68|-480570.59|-1334633602495.500977|true|nap time|ulysses quirinius|2036-02-23 15:54:13|2068-06-07 20:02:06.356997102|2088-09-29
    +-59|-28646|1797164732|3949|18310.99|4375064.78|2366334406338.168945|true|american history|holly thompson|2027-01-27 17:59:41|2045-11-11 14:23:14.608629099|1987-09-11
    +123|16767|1198701102|7762823913046556672||1643132.07|2312892585063.259766|true|linguistics|sarah underhill|2017-01-27 09:35:20|2035-06-28 17:21:09.561662434|2086-02-05
    +127|-14936|1772349172|-7884460946615984128|-20111.34|-2887093.65|-2318194071335.035645|false|history|victor van buren|2018-11-16 12:15:48|2065-12-31 08:22:47.150581738|2100-05-20
    +18|2612|1373871781|-7255010240787030016|49301.50|3253928.69|-4736789216462.362305|true|biology|ethan ellison|2049-01-22 13:36:56|2015-03-06 18:11:17.822359715|2103-01-05
    +109||1647411522|-7262384251828518912|3972.39|154588.69||false||mike ellison|2015-10-15 23:12:55|2063-04-26 00:01:09.31563180|2026-05-22
    +107|-2027|-1856034030|9132009829414584320|36330.70|2284543.68|-3148453530441.202148|true|study skills|gabriella nixon|2031-01-12 14:14:17|2050-02-14 06:21:51.178052619|2016-01-05
    +7|-12626||7573530789362262016||-1103225.79|-3059007558357.649414|true|industrial engineering|katie robinson|2032-10-19 18:30:33|2043-10-22 16:20:40.509237844|1984-06-13
    +49||-36038293|7157247449513484288|||3344159968330.622070|true|geology|holly allen|2072-05-19 04:19:02|2064-02-17 13:02:26.377162854|2082-10-30
    +37|-26666|-890552359|3493|-41018.63|-3192687.31|610319388952.653320|false|kindergarten|sarah young|2066-06-27 21:59:53|2041-09-23 12:15:37.914672228|2101-03-11
    +-43|-12811|1265528735|7491898395977523200|-17199.24|2696923.64|3366546613003.044922|true|undecided|ethan robinson|2029-10-16 04:28:37|2069-03-21 18:01:09.359881056|2039-11-24
    +126|2683|-1462604138|-7276111129363046400|12788.08|-3553203.52|2029170799165.562500|true|linguistics||2061-01-10 10:34:14|2036-06-23 07:16:14.299460550|2072-11-12
    +-109|23031|1517488324|275|30188.31|-3437436.90|72393656889.993164|false|forestry|luke quirinius|2073-09-17 10:02:35|2058-07-18 11:06:36.866728379|2072-05-24
    +-119|-21412|962712814|2715|-42015.52|-3416767.70|2730111077795.705078|true|nap time|ulysses brown|2031-03-11 11:56:59|2035-09-11 18:57:20.545053849|2009-02-02
    +-109|-32022|793047956|8150115791664340992|-13856.97|3208380.07|3846344885594.810547|false|education|mike laertes|2037-10-15 17:01:28|2070-05-09 00:38:50.111446261|2025-09-01
    +-1|-20591|972835688|7844258063629852672|-30899.74|2941710.70|2520339817545.540039|true|joggying|victor quirinius|2053-05-10 11:05:39|2025-06-02 14:48:41.118807009|
    +-86|-28097|291866793|1095|-13029.79|2395802.44|4638765048183.728516|false|debate|nick hernandez|2064-08-23 06:30:52||2103-11-18
    +|16439|1972940844|8759089349412847616||2775872.41|2078988628132.627930|true|philosophy|nick quirinius|2020-10-01 01:39:49|2046-10-08 04:45:36.155827749|2046-06-24
    +8|-7230|-2065080832|950|3517.45|-355907.54|-3463587144624.789062|true|biology|priscilla falkner|2077-07-28 16:36:07|2034-01-22 01:11:00.675336432|2051-01-03
    +50|-11769|-558456218|6960137166475911168|1455.89|-3580235.81|-2225358531562.626953|true|undecided|yuri xylophone|2028-03-16 13:31:23|2036-10-04 05:21:52.917455779|1983-01-13
    +-92|1558||-7612455481940246528|20027.51|4940501.37|-2309132393323.310059|true|american history|alice king|2023-02-04 21:40:55|2072-10-28 16:02:05.785926481|1980-05-03
    +-110|18346|-1603374745|7471208109437304832|2980.46|-4262671.69|-3375577964698.521484|true|philosophy|alice zipper|2061-06-20 11:36:22|2080-05-22 04:06:26.257944476|2080-04-10
    +-28|-21648|1081920048|1501|-39377.99|3185244.69|1714008069386.992188|false|values clariffication|tom thompson|2063-04-15 02:39:09|2075-09-24 04:37:07.641525192|2084-05-31
    +26|29245|-817093900|-7145585429014888448|29563.03|2338619.47|-47947487681.199219||geology|priscilla polk|2015-11-23 03:51:26|2048-05-27 06:59:27.15089547|2053-05-23
    +90|-3222|1301426600|7794244032613703680|43143.95|4941656.20|3845904537462.523438|true|chemistry|irene hernandez|2054-10-20 19:50:57|2075-08-30 11:18:34.922622163|2047-02-03
    +-35|27983|975932228|-7510418793070075904|-31952.67|-547797.70|4033293860789.123047|true|geology|victor carson|2042-12-09 03:28:14|2065-01-16 03:24:08.971263541|2010-06-30
    +-14|-29239|230954385|7983789401706094592|28755.14|2806157.04|-3764755036302.638184|true||wendy robinson|2022-06-26 13:18:28|2033-05-18 03:11:50.871005511|2007-03-28
    +-86|-15393|-1392487784|3990|25365.72|-3122775.81|-3136333430552.511719|true|study skills|irene carson|2075-08-29 11:01:05|2058-01-25 11:54:54.863071401|1985-03-10
    +58|-31551|1443426396|3830|-44944.38||-1901078252690.346680|false|mathematics||2069-02-25 02:44:41|2026-03-16 18:07:16.176024280|2087-02-16
    +-79|-12463|-37773326|-7152177800841502720|15979.17|136523.07|4328797369585.437500|false|zync studies|fred hernandez|2074-01-21 22:42:03|2063-04-17 17:43:27.97971264|1986-09-21
    +-125|-29323|44009986|1856|36983.95|841608.25||false|history|rachel young|2052-06-19 09:14:41|2025-12-14 16:53:02.735673012|2076-10-09
    +-62|-28146|-901778330|2712|-29272.48|495232.86|-4971306749304.673828|false|values clariffication|david young|2023-08-22 20:04:18|2018-02-08 01:53:45.121138509|2067-01-28
    +26|-24538|1767359228|-8746702976270385152|-10262.64|3827137.43|738589831875.913086|false|wind surfing|zach brown|2013-07-02 04:00:34|2039-08-29 12:57:02.351355581|2047-07-31
    +21|7128|-217930632|-7903158849011843072|777.32|-3589619.96|424579951542.725586|true|philosophy|rachel robinson|2046-01-04 01:02:07|2066-12-14 05:45:23.618651463|2006-08-23
    +-87|19739|-1699049982|2244|4616.60|2733769.47|2986145815124.956055|true|linguistics|yuri carson|2043-05-07 16:06:28|2076-01-07 23:49:22.432671324|2089-08-31
    +66|29834|1307148254|2735|42117.25|1129420.19|4620474372066.283203|false|kindergarten|jessica xylophone|2051-08-08 09:05:51|2048-06-06 23:50:37.973877723|2060-05-08
    +77|21511||1168|47905.76|-2302110.36|-2332600454045.881348|false|debate|mike van buren||2046-10-07 14:52:17.37357221|2001-12-10
    +-31|-9482|1330219997|-9080956291212132352|26375.90|-1329189.97|1655214180391.766602|false|chemistry|holly xylophone|2051-07-18 20:15:33|2058-08-16 06:28:29.740534617|1982-12-07
    +-43|27454|219415594|945|-12127.36|4216836.73|3646917581760.433594|false|education|calvin steinbeck|2076-07-10 08:53:35|2018-10-06 03:52:24.596896729|1983-09-04
    +55|-24267|1458051497|8417381121663746048|49672.46|2222816.47|-3645327331789.474121|true|yard duty|wendy garcia|2036-06-01 21:18:52|2018-11-11 07:00:13.274801916|1996-09-30
    +-13|-32541|1563120121|3823|28411.76|1170745.89|2011473770962.695312|false|undecided|nick polk|2023-03-23 06:07:51|2076-10-16 02:23:25.407392909|1985-12-24
    +103|18395|-2144138362|-7868306678534193152|21455.88|-4870777.55|-2770023875366.263184|false|american history|victor laertes|2047-01-10 23:37:37|2032-02-23 13:39:32.775684381|2052-11-07
    +-23|17964|-1100641049|-8518060755719585792|42499.60||-3519839102751.412109|false|chemistry|luke miller|2015-03-11 11:14:33|2068-07-24 11:17:54.14270195|2080-02-08
    +20|32096|1796950944|3703|6669.34|141075.74|-2930607649235.206055|true|wind surfing|ulysses garcia|2013-02-18 21:06:48|2015-01-14 11:10:53.143208292|2030-08-16
    +-60|-4804|-1988508336|7027529814236192768|36634.88|292742.80|-4785713903272.927734|false|kindergarten|bob young|2028-02-08 13:16:09|2058-06-30 12:21:12.174769417|2103-08-30
    +-31|-15633|1109664665|-9105701280936501248|-7200.23|-3290965.94|-2858696088737.069336|false|wind surfing|nick king|2071-07-06 14:12:56|2072-05-07 22:23:28.909242175|
    +50|28118|-432218419|7570474972934488064|-23669.50|4714268.20|3257421256704.163086|false|undecided|nick white|2016-04-08 07:34:48|2041-04-06 04:08:01.271192129|2075-07-29
    +-64|23229||-8430370933326536704|34019.68|1961397.66|2639849725646.250977|true|religion|luke carson|2079-03-12 02:12:51|2077-01-08 09:54:43.763841247|2004-07-25
    +-49||152891873|417|||4308469366197.300781|false|forestry|irene underhill|2079-06-08 05:52:56|2029-11-07 18:19:49.927077461|2033-09-20
    +-47|-20729|1673218677|49|37640.59||-713961786513.870117|false|debate|xavier falkner|2020-04-17 16:54:30|2028-06-03 04:57:19.609929160|2082-08-25
    +-16||-1733560816||24701.68|-4273424.19|-2584537633556.181152|true|biology|david davidson|2053-08-30 09:50:24|2051-05-28 08:21:28.136207105|1991-03-05
    +55|-24186|-1871209811|7068517339681259520|46423.34|2538160.45|-3739128624481.580078|false|yard duty|fred polk|2038-03-02 19:52:08|2066-12-04 13:20:32.335880942|1994-06-21
    +-8|26625|-906986958|2187||2742196.26|-4507463460711.384766|true|history|wendy johnson|2064-06-20 11:56:49|2025-07-17 23:27:04.166297070|2052-06-20
    +30|-12880|-40407627|2348|-6268.82|2266879.88|3002655520364.605469|true|topology|quinn underhill|2063-07-19 09:45:06|2045-02-16 17:39:10.146220452|2075-03-15
    +-85|-15762|-1556127172|-7139677575412686848||-4483436.56|-1265174168415.015137|||fred nixon|2061-03-21 05:44:18|2014-10-08 04:02:23.556239597|2071-03-05
    +-6|15862|1012696613|888|-22200.62|3639449.27|2198173643342.400391|false|biology|david hernandez|2051-03-03 20:56:40|2058-12-04 08:26:03.321045989|2027-01-07
    +92|-23550|1238986437|921|49411.28|1421037.00|-4845225760189.179688|true|history|ulysses underhill|2036-05-02 17:20:37|2018-07-15 18:49:56.958391676|2092-11-15
    +4|18984|-267130580|-8034414142083170304|34672.91|-4128287.64|766368935100.487305|true|education|fred steinbeck|2079-07-18 08:49:16|2045-03-01 15:48:34.880096673|2053-04-30
    +54|29015|-1065248998|-7330203470474985472||-3027947.92|3399052323913.345703|true|biology|sarah ellison|2026-03-03 11:12:47|2057-08-18 20:13:54.240194383|2059-10-09
    +-119|31140||789||-940868.71|-4363934228329.320312|true|quiet hour|zach falkner|2049-03-25 02:38:35|2041-11-22 05:46:48.4658675|2006-04-06
    +-127|-31998|-71305062|-7158472098920390656|20591.34|3729248.23|-526374280413.826172|false|mathematics|ethan young|2066-06-15 04:16:02|2030-06-20 07:00:32.309007556|2006-02-26
    +51|-1034|-103219371|-7881262505761710080|29627.76|712784.66||true|nap time|xavier zipper|2066-05-06 03:31:19|2030-08-29 00:39:40.299687444|2071-11-21
    +-113|26241|-971203543|8451612303224520704|-29948.96|3327680.13|-18523066115.392578|false|nap time|quinn miller|2065-09-30 09:42:56|2037-03-26 02:34:03.815937235|2026-03-17
    +104||-996953616|3541|18791.19|-462147.25|2541474056133.629883|false|philosophy|victor zipper|2060-05-20 14:26:32|2068-07-18 13:21:40.858914884|2085-03-29
    +37|2784|644934949|8143462899383345152|19598.55|-4115146.61|-1524392874934.043945|false||irene hernandez|2047-08-20 09:34:21|2025-03-27 03:58:40.290018987|2096-08-25
    +|-19276|605141554|8383159090746204160|45522.67|-4427581.33|-3551912779492.703125|true|linguistics|luke xylophone|2030-07-10 13:00:24|2048-12-11 23:41:59.785955383|2035-02-18
    +-32|-24178|1148500740|-8923529803981905920|-36565.06|3529368.28|2380664601539.273438|true|topology|luke thompson|2073-01-23 21:54:44|2017-08-29 15:18:43.706057935|2022-08-28
    +|-3813|1992977592|7885697257930588160|40610.97|4059042.75|-614948955390.488770|true|study skills|quinn garcia|2071-12-14 10:05:04|2022-09-08 09:37:06.875565982|1975-07-07
    +30|10615|-847235873|9107991000536498176|-9874.16|-4892978.32|-830926430811.871582|true|values clariffication|oscar young|2031-07-31 13:47:34||2060-07-26
    +13|-15389|865013617|1409|-45007.46|1282299.74|-3272482088828.603516|true|joggying|fred miller|2070-08-11 19:19:39|2075-09-01 02:16:50.347835856|2078-10-27
    +-96|11809|-1644966759||-14002.93|-4469818.97|1917126159337.136719|false|biology|calvin quirinius|2045-03-07 23:39:15|2032-04-12 02:27:12.657670943|2009-05-05
    +20|6372|1478365409|-8675892979328212992|38991.14|-2146326.91|-1669697999836.649902|false|joggying|calvin white|2027-05-03 14:40:13|2065-12-20 12:05:02.853602253|2009-06-25
    +81|7299|-1126628450|8920344895701393408|-21909.86|-1392050.93|-3312751733215.968750|false|opthamology|sarah hernandez|2015-04-12 04:15:23|2080-04-01 07:17:45.74683010|2103-02-26
    +-61|26540|-968377273|-6974654664348033024|-26186.32|3944430.54|1407237264653.069336|true|zync studies||2049-02-15 00:08:46|2021-07-04 09:11:18.595023146|2043-04-25
    +|-13973|1912175355|-8879742387365429248|-24520.99|1718891.92|990917459853.477539|true|yard duty|alice nixon|2022-01-20 13:23:39|2035-08-16 20:58:12.262393960|2095-12-31
    +-80|-15299|1318606691|-8938849835283677184|-8804.48|-4963660.13|-4051044335853.006836|false||quinn young|2069-08-28 15:39:57|2031-10-23 14:35:30.465215236|2027-10-23
    +-33|-32462|1222935237|-7213775605408178176|9353.59|-2865973.47|-29816572801.699219|false|study skills|gabriella garcia|2073-11-24 15:12:21|2020-10-06 02:54:29.617493343|2018-02-28
    +-125|22704|-1096771844|9199741683232399360|28595.90|3955962.84|4910989073004.792969|false|philosophy|ethan van buren|2062-12-30 06:36:02|2054-01-25 08:11:43.748458487|2081-06-29
    +-22|-18659|1416850873|8489584373231919104|34567.10|-2690728.31|-4902808539333.527344|false|linguistics|bob polk|2071-07-20 03:43:45|2059-05-11 00:01:57.809402055|
    +-77|-19912|-340951385|-7344947507044466688|12056.25|-2933320.14|4292999497091.330078|true|philosophy|xavier robinson|2042-03-04 00:33:10|2081-01-29 00:05:29.43863329|2014-02-25
    +-101|-13743|-158420748|-8756989568739835904|30210.95|1930694.98|-4980390206441.782227||mathematics|ethan davidson|2075-07-09 02:02:31||2004-11-05
    +97|-2152|-1616030844||6923.19|1009414.39|-4803916908307.214844|false|opthamology|irene falkner|2019-05-12 23:35:04|2034-10-21 07:24:20.465428653|2083-09-11
    +96|22603|925032386|-8046189486447017984|-8322.37|-3646620.83|3514393344286.617188|false|study skills|david polk|2061-10-25 22:52:55|2047-10-20 20:43:26.170125012|2094-08-30
    +70|4491|-1362178985|1287|-45257.20|734363.29|-1819407181812.877441|true|values clariffication|quinn laertes|2015-06-04 07:45:25|2078-03-22 02:23:35.763695497|2012-10-25
    +-98|-18140|-835198551|982|39755.57|1372628.14|-1714648472726.310059|false|kindergarten|rachel van buren|2072-04-30 23:06:10|2076-04-03 10:07:19.979852554|2058-01-21
    +50|23956|2097519027|615|-24059.46|631764.92|-2495415685760.988770|true|forestry|sarah laertes|2028-12-22 13:28:33|2017-11-07 21:48:52.848530518|1993-12-02
    +79|20872|1495575878|-7542857121910046720|-11668.81|-4226007.73|-2260276582420.459473|true|american history|quinn king||2044-03-22 10:07:42.593619051|1970-06-19
    +65|-24591|-496870819|1648|-200.89|-3053285.53|-3512079952779.068359|true|mathematics|tom quirinius|2050-08-20 15:12:10|2020-12-29 05:36:09.651652502|2053-03-08
    +-68|-29685|-1127100849|-8203008052020879360|37283.21|-1596933.52|4832262437578.585938||geology|oscar white|2048-02-17 07:53:44|2033-11-12 06:12:45.238117579|1973-05-13
    +|-3206|-1741895392|-8688153842294595584|-2854.09|-1738342.95|-4557913856473.696289|true||rachel underhill|2077-05-20 02:03:40|2033-08-05 01:49:40.305347119|2027-10-12
    +-98|16734|-596963345|-7707867749256445952|-47352.27|-4521296.31|3236784579167.704102|true|education|yuri ovid||2077-01-28 08:26:13.323779922|2093-03-26
    +-66|-24368|1832650234|-7894382303337832448|2348.21|46896.52|-888063795956.135254|true|mathematics|alice garcia|2061-05-29 01:02:47|2018-08-11 08:18:59.362818129|2057-03-18
    +-5|9716|-499533481|1667|44403.36||2674989060827.001953|false|history|oscar underhill|2055-10-25 00:02:32|2069-02-01 10:23:16.84520652|2053-06-24
    +48|-9183|-607285491|8780196485890555904|75.56|-3870004.21|662312551004.902344|true|yard duty|sarah thompson|2076-06-19 08:17:36|2013-11-07 19:36:51.34765657|2045-10-15
    +55|-15109|56316391|-7404057145074712576|2893.22|-4203203.26|-3201041913797.339844|true|kindergarten|sarah polk|2044-09-05 22:10:18||2093-11-05
    +-69|-20946||-7433265617153343488|-47857.01|1123959.85|-4751032804232.125000|true|religion|gabriella allen|2046-02-04 08:22:40|2021-11-15 18:22:18.608780184|2069-04-29
    +7|-1180|-423378447|1127|-30027.48|-1414107.87|4302554900117.384766|true|zync studies|oscar white|2020-12-23 10:18:38|2050-09-21 09:03:11.214720790|2099-04-24
    +11|14376|234452496|-9149719074367946752|41986.01|4484955.64|-1431661810573.672363|false|wind surfing|victor robinson|2077-06-15 17:08:48|2020-07-26 05:43:36.11096510|2083-06-06
    +-31|-408|-1933374662|763|5026.31|2220778.49|-1880721142952.310547|true|values clariffication|priscilla ovid|2079-11-27 10:31:13|2078-12-02 09:20:02.449941920|2020-09-15
    +-124||206942178|3249|44696.56|-2844962.02||true|industrial engineering|zach miller||2014-01-24 20:41:26.351041828|1972-03-05
    +74|-17840|307333276|-7708932208121225216|33649.41|-2209443.33|-3174835614765.178711|false|opthamology|wendy polk|2049-02-15 09:49:07|2021-09-17 10:39:35.795144931|1975-02-26
    +35|6015|-1343327|2689|-34762.90|-3352811.58||false|nap time|oscar king|2061-03-02 03:38:49|2054-04-17 12:47:53.423961154|2027-06-04
    +14|-20180|720703232|-7881351200983613440|-14311.83|1718525.86|4502617863309.339844|false|xylophone band|oscar garcia|2035-03-09 12:34:23|2019-05-05 02:57:22.203595007|2072-11-12
    +-17|26296|344989592|-7515996202498473984|-29958.06|2161214.73|-209259678055.531250|false|philosophy||2068-02-10 07:56:40|2042-06-02 13:15:00.744949321|2074-05-17
    +-111|13488|929751599|1681|15142.21|2124532.47|-4344059773498.542969|true|zync studies||2061-05-05 09:51:27|2035-01-03 11:31:13.490764419|2067-09-16
    +-119|28064|631711489|-8507279516485566464|46368.28|1381513.63|-3198785922561.029297|false|philosophy|victor johnson|2053-09-16 22:56:54|2064-09-13 02:14:41.203564754|2052-09-23
    +-3|-5897|997193329|1316|-36632.56||-4035360861641.272949|false|xylophone band|david johnson|2033-08-30 22:43:18|2021-08-20 07:30:06.17481138|2093-04-19
    +22|-22922|626941809|7892026679115554816|-24481.02|4819862.86|3630292090950.355469|true|philosophy|victor ellison|2072-04-15 13:45:41|2069-12-27 19:14:43.4196486|1970-10-03
    +|-22534|-1412187081|2279|35125.32|-1204413.83|13572776973.802734|true|history|gabriella robinson|2035-09-28 15:15:33|2079-06-21 21:50:11.269734623|2021-08-31
    +8||842283345|873|44738.40|4982540.22|2545702719622.049805|false|chemistry|gabriella underhill|2044-03-23 02:31:44|2063-03-07 17:44:02.4074809|2059-06-02
    +96|-5699|-163859725|8142241016679735296|-21581.78|-1551671.29|-2922841514857.416992|true|wind surfing|zach laertes|2014-12-09 06:23:35|2026-11-05 11:43:02.534377887|2023-06-06
    +-2|-26128|1513689502|-7811060170911375360|5804.81|4404088.96|-3923411784468.591309|true||bob allen|2075-07-07 20:40:56||2096-05-07
    +13|28551|-1568646283|8411494452500930560|-34399.96|2787253.76|-1592892075417.431152|true|topology|rachel brown||2025-05-06 23:15:59.141775309|1993-08-22
    +52|40|587206979|2803|-38274.78|-3013975.64|4769880713478.898438|false|joggying|alice zipper||2062-08-30 07:04:29.405219012|2075-12-10
    +-77|-13238|-866304147|1247|30977.58|-1455951.83|-379569461462.274414|false|forestry|holly young||2060-02-22 09:39:53.875517714|2054-12-02
    +-16|13154|1141303816|-7673901622181953536|32011.03|-2072819.72|2331383540354.394531|true||quinn xylophone|2013-06-15 10:55:00|2066-12-15 17:24:21.774901994|2038-05-15
    +-94|-21451|-203416622|268|33257.23|3858254.19|4841902350857.658203|true|geology|nick miller|2030-02-09 07:34:05|2061-02-09 17:09:30.792720771|2026-01-05
    +32|13150|2142592987||21739.73|443657.37|1617671196062.850586|false|values clariffication||2055-04-27 17:42:09|2035-04-30 09:03:34.834154256|2048-07-11
    +85|-1767|-1977762695|-7195217207163166720||2897773.78|4178958976210.437500|false|values clariffication|oscar quirinius|2071-10-09 18:35:28|2039-08-28 00:08:15.223157462|1976-06-27
    +-1|-19464|538766635|-9012093603044245504|16783.69|102999.04|-3761561711229.433105|false|yard duty|irene laertes|2039-01-07 08:49:02|2077-12-04 00:37:45.588821251|2044-03-15
    +84|7569|1739911574|8920533610804609024|39584.58|-2087360.22|2573136281954.120117|false|quiet hour|ulysses ovid|2058-06-30 06:37:18|2028-03-12 18:17:11.549298490|2073-09-23
    +93|-254|1159353899|455|-26164.13|-2925253.46|-3708898652238.446289|false|debate|ulysses polk|2071-12-06 03:25:24|2027-01-09 04:39:00.318567529|2103-06-02
    +-77|30619|1370723240|9148071980848742400||2115468.25|-2676483804058.200684||education||2076-06-26 22:59:22|2068-09-02 06:39:15.162172013|1981-02-07
    +-76|24512|1316931||43602.63|-4235241.77|-2225959074468.946289|true|study skills|priscilla brown|2069-04-16 14:07:02|2059-05-03 07:04:57.531362893|2040-08-20
    +|-17607|-628790799|-7093825013581979648|41798.75|789629.23|4045302222731.130859|true|topology|holly nixon|2034-09-29 04:12:38|2028-09-01 15:46:24.993528329|2040-12-14
    +-48|18142|1203482872|1342|-48524.80|1488860.19|1080708792025.255859|false|geology|david underhill|2038-12-12 20:41:41|2054-06-04 10:17:24.350972371|2048-11-22
    +-35|-28366|1182646662|7295926343524163584|37147.82|-17339.38|850711115335.975586|true|religion|fred ellison|2056-12-16 18:57:16|2030-08-08 13:20:23.87227693|2061-10-03
    +|-11187|2127682701|7874764415950176256|21152.51|-1228307.04|-3933963121027.943359|true|quiet hour|sarah garcia|2061-05-01 14:39:07|2039-11-30 14:02:00.787382842|2063-08-30
    +96|-25282|2009215103|8682955459667951616|-30776.76|-951920.19|339774798412.620117||philosophy|bob thompson|2018-01-25 07:52:23||2072-09-01
    +75|12802|872554087|8825059717746376704|-23535.33|-3969818.94|-2914303137233.673828|false|wind surfing|jessica xylophone|2024-10-12 09:03:29|2080-11-22 08:10:16.820114627|2023-11-21
    +-91|-31335|-1575588203|1651|530.17|-2153555.94|-2020396257019.356934|true|history|calvin zipper|2036-06-05 07:17:08|2077-07-11 08:39:35.486363899|2097-02-12
    +123|-25183|69110370|1187|18383.16|-1514304.51|2213587444204.989258|true|undecided|fred ichabod|2027-01-18 08:53:58|2053-06-12 11:25:39.320312625|1998-09-27
    +-66|21168|976870621|3197|-21478.88|1822183.11|-2601260542275.482910|false||calvin polk|2030-09-09 10:31:01|2071-07-11 10:12:44.480141278|2040-12-04
    +108||-144862954|1115|37537.67|-3636489.74|-2775808925857.248047|true|values clariffication|sarah ovid||2045-02-17 17:37:23.51474731|2048-11-26
    +-116|22118|1117805438|8945302550165004288|37582.57||501497065597.860352|true||calvin zipper|2066-05-15 01:52:49|2046-09-19 21:50:44.293786457|1977-03-05
    +107|-23153|1571267481|9114850402293882880|-30178.38|1989319.92|826991484976.567383|true|nap time|gabriella underhill|2046-10-28 01:02:40|2045-01-12 05:45:37.921528799|2014-07-18
    +97|12327|1603612975|8935252708196999168|16046.26|-67740.16|2793889545913.791992|true|opthamology|gabriella steinbeck|2043-09-23 14:40:09|2078-01-01 15:35:48.152129849|2031-05-27
    +39|-16820|43983130|2663|-27389.47|775624.70|-1775814976651.695312|true|chemistry|luke allen|2021-03-30 12:39:17|2043-02-02 21:34:00.252214409|2080-09-28
    +-73|-15866|-99205196|-8535957064499879936|-48198.87|-2414154.16|349441023585.761719|false|study skills|priscilla hernandez|2055-03-14 23:42:22|2027-01-12 03:52:14.47123348|2080-01-02
    +70|-25463|748185058|9091082386452684800|48356.67|-3526156.73|1250977516314.754883|true|industrial engineering|alice quirinius|2015-09-13 22:17:45|2072-08-09 11:39:30.710221441|2052-04-27
    +45|20124|-470798506|2915|43588.39|-762457.78||true|forestry|wendy carson|2072-06-06 06:46:18|2014-12-01 04:17:26.513106497|2055-12-01
    +28|-9196|-1947868215|2183|49404.36|-3414112.11|2109147570161.909180|true|geology||2027-01-27 11:49:05|2061-05-01 06:27:32.791222399|1990-10-25
    +14|25381|659397992|-8147405381260345344|47303.15|-3806020.32|997601888271.546875|false|values clariffication|fred brown|2074-06-25 18:50:23|2057-10-25 22:57:33.478783182|2105-09-25
    +82|-1367|895763504|-8654797319350927360|1232.14|851336.52|-3820905742244.491211|true|religion|bob steinbeck|2032-12-27 16:52:15|2041-10-19 14:13:28.458891032|2104-03-31
    +18|-13466|-343173797|1261|-24504.59|-2671371.43|4817498142971.957031||linguistics|fred ellison|2053-12-16 04:23:07|2021-12-04 13:24:23.478621300|
    +113|-29988|-213198503|8775009214012456960|29118.01|-2672928.16|3597735313714.380859|false||holly allen|2038-04-29 17:39:45|2035-02-10 07:52:53.120396795|2007-11-09
    +126||1844415080|2810|-43710.73|4357967.19|-3835582850357.113281|true|history|holly xylophone|2056-04-30 00:14:07|2037-07-23 02:43:53.584900523|2038-06-02
    +126||43672187|9067985867711291392|8608.12|-3919341.08|-1433926422321.541016|false|linguistics|alice johnson|2040-06-19 19:13:38|2032-01-20 11:14:11.710454857|
    +18|-31663|-1721368386|-8240684139569233920|-41884.25|-4050155.29|-3799361430049.685547||linguistics|wendy brown|2067-02-20 18:47:51|2075-11-01 10:27:41.797075523|1973-08-26
    +-115|-20828|-1477897348|1719|19183.72|3667748.43|196163723446.253906|false|philosophy|nick ovid|2038-03-16 23:59:43|2031-04-04 05:56:43.142902096|2094-02-12
    +-61|20411|2076370203|7917494645725765632|34656.95|-1033805.16|-2293878051389.661621|false|education|bob thompson|2041-05-09 00:13:16|2064-12-30 17:41:02.427765133|2096-07-09
    +100|410|2057486961|3622|49893.07|4038206.70|-434822164725.155273||linguistics|alice laertes|2059-12-30 17:21:06|2067-10-16 22:22:23.909102811|2036-12-30
    +68|26697|-1079633326|-8866442231663067136|-43306.72|-616547.34|-892751806585.678711||opthamology|katie robinson|2066-09-01 03:27:00|2059-11-14 17:00:11.431792872|2052-04-27
    +6|28146|-309571354|7871554728617025536|10210.31|2684254.99|717298056153.232422|false|nap time|gabriella steinbeck||2079-10-05 02:45:16.165020579|1988-02-01
    +-94|-22554|-1114208576|-7380731416973295616||4566778.78|4025279026650.033203|true|xylophone band|jessica hernandez|2060-08-11 01:49:22|2019-01-16 09:25:47.428200312|2038-11-22
    +9|11059|1114521964|-8603817012434198528|38309.84|-2121451.24|-1723505323178.535645|true|values clariffication|victor garcia|2076-02-21 16:02:29|2024-12-31 21:24:49.835872918|
    +-34|10425|-45439614|-7192529627893858304|21468.82|666967.79|-1710357667596.511719|true|history|david ichabod|2051-07-14 18:03:02|2017-02-27 04:57:13.944522320|2050-01-25
    +36|31099|-1108723753|-7287583262310350848|-61.04|-2662929.33|3709449843031.081055|false|industrial engineering|luke quirinius|2036-01-13 20:52:17|2062-10-11 17:14:59.629263445|2045-02-24
    +113|-22915|-1352545619|7149417430082027520|-6486.11|-3357464.95|-1319020386301.517090|false|chemistry|yuri carson||2051-08-31 14:41:56.890230740|1992-11-06
    +-45|15011|-1061859761|7237310132329488384|-39128.71|319025.53|-1778680361202.553711|true|mathematics|victor nixon|||1998-02-21
    +-18|31370|1204834275|-8659643752269242368|39007.50|1149036.85|-2714226684056.946289|true|american history|katie thompson|2027-06-17 03:56:45|2070-08-03 17:42:35.323901256|2099-12-02
    +|20969|-469749219|-8051395538179063808|32233.91|2215432.29|-3014965054465.008789|false|wind surfing|calvin underhill|2014-12-17 19:52:13|2022-11-29 17:49:15.855148488|2066-10-20
    +-68|-4142|1488440165|1674|-20812.84|253603.39|-2105480401864.810059|false|quiet hour|irene steinbeck|2067-02-13 02:16:25|2022-12-16 07:14:39.23664527|2046-07-15
    +-68|27630|1575300276|-8813211231120031744|-14758.39|-1926059.11|-2833925266574.155273||chemistry|ulysses allen||2032-05-08 18:36:19.400564422|1988-01-19
    +-106|25835|-533227056|-8103788088118018048|-549.34|4959162.61|4308090638361.007812|true|kindergarten||2073-10-28 10:37:18|2079-07-06 02:03:22.207944103|2028-04-09
    +-57|-4726|814544198|7049773031131283456|10513.01|2178130.57|-2416479446500.758789|true|biology|wendy johnson|2020-03-30 21:56:57|2029-12-16 16:34:42.572182352|2097-11-09
    +-80|21784||-7035132060308643840|-6419.44|-4383105.22|-2020072615248.374023|true|philosophy|xavier hernandez|2018-09-25 15:18:36|2031-06-28 02:55:11.31963432|2068-08-05
    +-78|13795|-1769423338|-7303847963918393344|43895.68|-3084959.79|-4175764579458.167969|true|opthamology|gabriella johnson|2043-10-13 05:10:09|2048-05-19 05:40:47.176953399|
    +-69|-16027|819069589|-8046238369820344320|-22162.98|-345358.25|674713800507.689453|false|linguistics|zach thompson|2015-10-24 08:01:09|2020-06-08 12:08:21.511690843|2019-10-03
    +47||2068018858|-8503573595507761152|-10699.81|-2421607.62|2965768864894.222656|false|undecided|rachel ichabod|2069-06-04 05:16:15|2061-09-03 01:12:25.916465543|2030-10-14
    +85|26579|-1371840597|8822384228057604096|-41991.30|-138391.14|1063729386321.732422|false|debate|jessica xylophone|2054-11-01 01:40:23|2069-12-06 00:11:49.576880733|2090-07-08
    +110|-9759|-136514115|-7998947380180819968|-17463.46|1862978.01|-3370842772950.304688|true|study skills|irene ovid|2059-11-03 20:21:53|2061-07-03 17:32:49.424965303|2015-10-17
    +-26|-10316|-71449585|-7057750467944931328|7153.05|-3062092.43|2726424274449.748047|true|american history|fred brown|2051-03-07 14:50:11|2027-05-29 12:10:20.647953531|2096-04-06
    +113|-22215|674547678|1862|-27979.49|2034087.73|1306104776811.629883|false|industrial engineering|tom johnson||2021-03-25 16:24:12.665579887|2101-08-21
    +56|8551|-733239404|8583916402383601664|47565.67|-4620642.81|-2470480735156.428223|false|undecided||2027-03-26 08:37:58|2079-04-18 01:15:58.35100799|2056-01-22
    +98|-26361|1489169773|2398|42357.85|-3934751.72|-4429578287144.977539|false|mathematics||2048-03-15 04:39:44|2033-11-06 05:22:43.819682351|1986-02-17
    +-32|-19330|2068538934|7487538600082554880|7664.98|4132039.93|-2902910424869.702148|false|geology|quinn polk|2041-11-01 21:08:54|2035-01-03 23:06:46.765787876|2071-02-14
    +123|-20409|115470151|8677794924343164928|12145.54|-4842892.80|400760479300.927734|true|xylophone band|mike ellison|2080-02-28 20:34:51|2014-02-14 03:41:40.325566176|1989-11-16
    +-13|31135|6526476|8708232769657815040|-14997.43|4694403.83|333471522081.652344|true|xylophone band|sarah brown|2021-01-12 23:04:21|2067-12-27 14:26:36.287488601|2062-07-17
    +51|-27259|1829544791|-7623359796281999360|-34844.45|-3126581.66|-2421986983728.814453|true|education|tom miller|2044-10-28 16:20:50|2018-07-05 05:25:15.270012699|2071-07-26
    +-95|-17690||8983912573761167360|31719.06|-3442868.07|-1937958357952.531738|false|kindergarten|irene ovid|2058-03-03 15:38:13|2069-02-04 13:07:32.271143352|1983-06-19
    +10|-30669|1590744669|-7263060340185194496|-30819.18|1534635.31|4148791909569.630859|false|kindergarten|quinn white|2027-07-01 19:44:37|2056-02-18 15:10:02.841254943|2091-01-11
    +93||-971698865|-7873753603299540992|21852.31|-2581251.56|1539283723979.209961|true|quiet hour|bob carson|2026-11-09 05:31:12||
    +-28|-18659||-7720966287634112512|20392.44|-2222926.64|1939030156067.396484|false|american history|irene allen|2032-07-18 12:44:48|2026-12-26 16:48:27.776120795|1987-11-12
    +-45||-40284975|1806|35653.78|1321025.73|898563412192.885742|true|mathematics||2070-01-12 03:23:36|2034-01-24 11:57:42.385238227|2057-09-03
    +82|-13539|1456367662|-8260340354454503424||1093063.49|4557861543446.826172||religion|quinn xylophone|2078-09-07 14:49:27|2025-08-16 09:10:36.887036096|2011-11-10
    +125|5469|889772203|2842|38632.60|-4727107.13|2741920281773.531250|false|joggying||2038-11-21 14:39:38|2040-01-28 08:27:19.830854683|2085-04-27
    +21|-20517||7069729473166090240|-14418.84||735618454615.902344|true|opthamology|yuri ichabod|2027-06-30 14:53:30|2016-01-03 18:53:34.457261740|1970-04-12
    +-96|-25734|-273937943|8688483860094599168|927.24|931681.37|4368902606237.980469|false|debate|sarah steinbeck|2023-10-17 00:33:13|2043-10-07 00:08:12.855001312|2038-03-10
    +-82|-23719|-318206520||-23783.97|-837555.18|1762959989546.857422|false|study skills|nick xylophone|2034-11-18 23:41:57|2052-11-30 19:49:07.414111749|2033-11-10
    +58|7697|-1437126017|7517159036469575680|-1037.26|-3799864.48|-1963988180877.791504||wind surfing|yuri garcia|2070-10-26 10:29:55|2046-12-29 13:30:24.655176498|2075-11-28
    +26|16312||2218|46235.86|-3813446.01|-4186362730051.698242|true|topology|priscilla brown|2055-05-14 21:50:40|2037-10-04 05:24:46.310622445|1998-02-21
    +-95|28923|-1769037737|-8996954350906294272|5131.31|-3335839.19|892560184430.806641|false|values clariffication|victor underhill||2051-04-05 17:19:55.107473640|1979-01-11
    +89|20014|945683736|2089||2623267.83|-3717730752750.725586|true|debate|gabriella quirinius|2078-08-08 13:57:39|2062-07-26 14:12:36.59165411|1999-03-25
    +-52|-12809|-1533934649|-8651641150831362048|48413.99|345690.54|2773050456079.104492|false|linguistics|calvin thompson|2045-08-26 14:34:17|2017-09-25 09:26:40.856875502|1979-12-19
    +-107|13792|-1212524805|-8322751250650218496|26905.38|2307830.54|-3744767732184.754883|true|study skills|tom steinbeck|2027-12-07 07:23:38|2030-09-13 21:30:52.282498698|2061-01-04
    +-65|21299|-1222897252|1495|-31920.07|-1723567.09|2996766542753.092773|false|wind surfing|zach white|2025-09-09 20:08:36|2066-06-07 03:56:15.873165726|1992-10-17
    +101|30222|1667594394|-7848043121524228096|4374.75|1994312.07|4481698153938.253906|false|wind surfing|jessica zipper|2050-04-22 23:54:13|2047-05-21 07:47:57.777923762|1996-11-24
    +94|23766|-511198293|-7506254246954500096|20950.55|-748117.50|4939615715200.673828|true|debate|rachel ichabod|2065-05-20 02:43:05|2026-06-12 20:30:30.941431912|2084-07-12
    +59|5445|1974939899|7487338208419823616|44886.11|-4889191.55|4911721874172.769531|true|mathematics|gabriella steinbeck|2025-10-07 03:42:36|2045-05-23 22:17:04.9265626|1972-10-12
    +-99|-29601|816439627|-7875953567586451456|41944.06|-1349023.71|-1099429516405.692383|true|opthamology|jessica polk|2039-01-01 04:30:38|2076-02-14 07:29:41.551319075|2097-02-19
    +99|-3318|-1017027298|1863|24757.93|4444192.30|-3601627653993.517578|true|joggying|victor young|2031-09-16 06:06:37|2050-03-15 17:15:50.541208509|2079-07-14
    +-27|-3540|133276416|3835|-506.62|1920067.41|412688526446.700195|false|debate|jessica young|2039-09-26 14:48:34|2077-07-07 08:15:18.365301835|2084-11-07
    +-68|11572|25644069|1870|24153.48|4976723.43|-137386195040.676758|true|zync studies|holly falkner|2063-06-20 10:43:27|2024-10-19 14:18:31.573357244|2011-02-18
    +89|16250|491016124|-7978782649203228672|-46524.54|1265452.96||false|opthamology|luke allen|2056-12-07 20:53:08|2029-09-05 03:42:32.368459710|2083-12-25
    +-16|-19213|1509573831|3960||3292048.84|-240494562170.781250|false|biology|wendy hernandez|2051-02-06 00:15:05|2041-03-28 19:17:32.296675612|2083-07-04
    +-56|-6518|-1478812842|-7663293054873812992|-24105.64|-1332349.52|-1890560158574.032715|false|chemistry|sarah robinson|2069-08-29 19:26:05|2046-07-10 19:51:40.969328493|1973-05-29
    +-24|28077|-234278308|898|6115.34|652033.34|-4372957414734.699219|false||rachel robinson|2043-06-18 12:08:10|2066-02-25 21:32:14.879009176|1986-03-28
    +51|-24313|-938342473|3728|4223.26|-4736632.77|-2080633933255.846680|true|biology|wendy thompson|2019-03-20 15:36:37|2025-06-22 23:33:35.838318924|2083-02-02
    +-21|-4539|1469775272|-8615168537390571520|-13079.77|-3313208.22|3092499128010.040039|true|topology|quinn falkner|2073-05-16 23:56:42|2017-07-20 03:37:48.334651957|2055-08-30
    +121|29828|-1426893312|3579|4812.14||586167842633.002930|false|wind surfing||2022-12-05 16:40:51||2069-05-27
    +-79|-5314|-234758376|8111757081791733760|35336.11|4023549.09|4664860898254.835938|false|mathematics|ethan van buren|2072-06-06 05:31:29|2068-02-13 01:16:52.645583397|2065-07-21
    +10|-27295|-1299159155|3111|-40569.04|289098.86|-3359213193119.252930|false|kindergarten|alice ellison|2013-04-01 23:26:31|2042-01-18 04:14:18.953737200|2037-02-15
    +36|6892|1090344463|2539|21579.89||-2076008548637.773438||biology|sarah van buren|2044-08-16 16:50:11|2078-11-12 05:33:45.47089985|2054-12-18
    +-78|10807|550594651|7497306924248834048|-15217.81|3701631.53|-1010442667526.625488|false|religion|nick garcia|2039-02-22 19:12:26|2058-01-16 13:10:53.407263384|2056-02-09
    +-42|26236|-2019287179|2619|13960.36|3980665.59|2745518512550.937500|false|wind surfing|alice young|2049-06-17 08:19:54|2042-10-24 00:38:21.665346021|2001-08-01
    +-90|-13424|631954352|1423|43085.76|1692557.60|2335092836258.532227|true|values clariffication|xavier ellison|2028-04-08 19:20:25|2050-10-22 21:57:15.666787161|2100-05-23
    +66|-1679|472901914|7652123583449161728|-45141.82|-189883.01|2005119271038.431641|false|education|zach miller||2046-04-05 20:49:51.831050123|2081-01-30
    +-61|-19291|936752497|7271786885641666560|6407.13|-2622282.33|-1364893278530.487793|false|philosophy|ethan nixon|2064-03-25 01:50:11|2078-07-14 10:27:03.640728827|2024-09-18
    +53|-950||1447|-46967.91|-2338643.42|-3010460855327.999023|false|linguistics|wendy underhill|2061-05-09 13:52:02|2052-06-04 16:42:35.115432702|2058-01-11
    +12|-19479|1112783661|2553|-13650.84|-1699487.46|-4635813668758.892578|||nick king|2019-03-23 10:58:33|2071-10-07 13:18:51.554325420|1978-08-21
    +-8|26869||8148211378319933440|45519.19|1788716.14||false|values clariffication|fred xylophone|2060-10-30 01:15:38|2071-02-14 08:59:08.385880302|2083-02-05
    +0|29498|1393262450|7450416810848313344|-47360.59|1197766.74|1948267796553.970703|false|quiet hour|ulysses johnson|2064-05-23 05:56:44|2064-08-16 10:07:06.357883907|2049-04-05
    +37|30056|131031898|2979|16137.50|1448922.16||false|zync studies|calvin allen|2078-08-01 20:45:50|2077-07-22 17:11:31.657495112|2077-05-23
    +117||1751468853|-7120456708338688000|-45241.05||4658251999119.685547|true|forestry|tom ellison|2069-06-01 15:56:17|2030-05-31 03:22:11.194126870|2033-08-06
    +-120|-31659|-2016985611|7242751359672631296|43128.70|-191280.75|-2180975729787.064453|false|undecided|calvin ovid|2081-01-06 19:08:32|2056-06-12 01:13:12.668973447|2007-12-19
    +126|-23667|2111462911|9083076230151864320|11873.69|4280564.29|2689053897530.797852|true|chemistry|katie zipper|2041-07-26 09:46:11|2052-08-09 23:10:15.912587493|2004-10-04
    +92|7066|1785455842|-8754992450211692544||356819.61|-319780935492.240234|true|mathematics|ulysses robinson|2069-07-31 07:42:34|2025-12-14 11:43:12.789221272|1989-03-12
    +-23|11788|278601840|1537|-3884.62|1500759.88||false|geology|ulysses king|2061-08-15 00:45:19|2052-07-01 09:00:11.170504787|2080-01-24
    +120|-27232||7845953007588401152|30784.40|3725404.09|-4223940428806.862793|true|zync studies|yuri polk|2046-03-19 03:26:49|2028-12-07 07:12:17.409326696|2103-03-11
    +0|22388|1145627305|9023663198045544448|-16680.30|-4179831.25|-4258847848573.274902|true|linguistics|jessica ellison|2056-08-10 14:54:29|2067-07-03 10:13:58.634487256|2039-05-11
    +-102|256|352214248|8900545829211299840|-8142.34|1762753.22|2953930180391.744141|true|zync studies|calvin allen|2053-03-31 14:32:12|2054-01-17 02:39:01.769490002|1989-08-23
    +104|-8706|-400501472|1866||-4687152.61|-4538289046733.225586|true|opthamology|fred van buren|2045-10-06 23:55:19|2063-11-06 08:47:29.289513742|2067-08-04
    +-74|-22241|-1216166764|-8965578088652095488|11351.19|13624.69|4203637057188.273438|false|topology|nick king|2056-03-27 03:36:35|1970-01-01 00:00:00.158770873|2039-09-02
    +-12|-10623|-445353909|154|24563.70|4730542.74|-2976281476329.525391|true|history|quinn davidson|2071-04-24 15:00:40|2016-03-02 22:10:50.657625756|1976-06-03
    +8|-32491|-1232183416|2905|-45693.51|3407626.79|-2909647656864.500000|true||sarah underhill|2038-12-19 06:40:41|2070-01-02 18:09:04.499401693|2038-01-27
    +41||2090496825|8693036785094565888|449.14|2113679.49|-1419899825383.019531|true|chemistry|wendy polk|2068-07-21 04:20:28|2042-05-24 13:18:57.760909140|2050-11-29
    +43|-17366|41063276|3555|-23520.95|-3021088.64|-2720359307901.567383|false|philosophy|wendy brown|2080-12-20 01:36:25|2065-03-06 19:37:09.328344492|2090-05-03
    +62|26288||3910|-10665.79|4113207.86|-4165972904406.558594|true|values clariffication|gabriella van buren|2028-01-28 10:55:38|2055-04-02 10:27:11.136514457|2005-08-12
    +-76|27523|-191899537|-7792903881635938304|-988.63|2575958.83|134807836030.148438|false|quiet hour|rachel thompson|2064-09-13 12:05:33|2068-07-23 02:24:15.415114292|2006-10-22
    +38|-15944|-1141801925|-8395998375405912064|37395.60|-2057132.64|-3432804499574.260254|true|geology|oscar young|2070-12-02 02:21:28|2022-02-22 23:45:54.417929187|2101-12-05
    +-9|32017|1367179645|-8503342882470019072|-32948.66|2596690.92|-4465574272510.082031|true|american history|fred falkner|2028-01-06 05:29:29|2034-09-22 22:01:59.220097380|2103-12-30
    +-37|8727|523289079|3946|-43342.52|2528654.53|-3334121644829.863770|true|history|gabriella ellison|2039-06-14 08:58:40|2023-05-10 22:56:14.107897590|2058-11-03
    +-16|-1749|209430502|2662|-27058.30|1085405.78|-478847345395.791992|false|linguistics|victor falkner|2026-08-15 13:38:26||2101-06-16
    +29|20270|-985817478|2848|-17564.28|-1085683.98|1245671080660.350586|true|philosophy|nick ovid|2077-11-22 03:06:05|2018-10-17 23:16:57.22955518|2059-05-05
    +47|15266|670667262|3974|48495.59|-4950157.96|2359236945727.296875|true|biology|wendy carson|2062-12-04 16:15:27||1991-07-04
    +|-19295|1677197847|8201491077550874624|41623.77|1350136.93|-1226174674712.830078|true|study skills|gabriella ellison|2016-02-17 02:12:04|2043-04-04 00:37:27.881714077|2027-09-11
    +|12722|930008274|2386|-2270.46|1951711.24|-2269419016241.016602|true|history|priscilla zipper|2023-03-20 02:05:11|2058-08-15 16:21:32.650056302|2082-06-21
    +102||-1112062809|690|-44270.30|-967065.47|4944863867873.259766|false|mathematics|victor nixon|2062-01-19 04:23:24|2034-11-05 03:07:33.846605261|1982-09-16
    +-100||1091736925|-8962547695651323904|18808.85|-2597624.19|633233154165.040039|true|study skills|yuri nixon|2027-02-06 11:15:50|2068-11-02 21:11:23.741655982|2094-07-19
    +55|-26304|1054864168|2227|-25135.64|-2339553.19||false|mathematics|jessica carson|2018-08-30 01:06:23|2078-11-30 02:36:40.607075532|2013-09-08
    +-92|29168|-560322190||22456.65|4966767.72|-4492662222754.084961|true|education|ulysses ellison|2026-06-11 07:20:10|2028-10-07 16:00:16.652808441|
    +127|-6294|-41242237||-24339.41|-4580152.68|-4150973486962.570312|true|kindergarten|rachel johnson|2025-04-25 21:40:13|2064-06-21 01:47:28.67945256|2026-04-28
    +-104|-25115|824836988|-7020852530219171840|2631.26|-1696924.47|2636281196281.515625|true|religion|ethan king|2064-04-06 11:56:42|2079-03-23 13:15:05.713728857|2012-06-08
    +82||373031319|6967631925774639104||-3338254.44|500376662732.691406|false|geology|alice johnson|2046-06-05 16:50:34|2033-10-31 22:49:14.847949456|1990-11-23
    +-6|-30482|1275228381|8145750910080745472|-32038.63|-2954981.27|2949286209907.953125|false|linguistics|david young|2035-10-06 02:29:21|2031-02-11 07:57:23.673629394|2105-04-02
    +113|-23323|-839512271|8994608999945125888|-28055.16|1643018.67||false|education|priscilla steinbeck|2054-10-12 02:43:21|2027-05-27 09:57:29.960419451|2005-01-23
    +15|-7024|923353533|3940|7872.92|-2081172.60|-2466850727047.962402|true|wind surfing|ethan king|2027-10-04 14:01:27|2049-07-15 07:58:03.472552275|1988-08-26
    +|-6283|-311437801|-8990843030306717696|-41276.81|1722079.14|2070485805872.195312|true|wind surfing|nick miller|2022-01-16 10:36:26|2029-06-16 15:44:00.263282479|1993-07-11
    +25|6980|-1892816721|-8182421179156905984|32052.05|2697304.50|-2452231930319.044434|true|mathematics|sarah van buren|2057-11-28 06:45:32|2065-11-02 04:49:45.252309410|1984-05-26
    +-25|26744|-1489628668|-7878145001776152576|38864.72|-543066.51|-3989748992325.772461|true|philosophy|tom quirinius|2027-08-03 17:06:47|2071-12-24 20:56:38.549176629|2093-04-14
    +-122|25038|100270148|9096395849845194752|16126.36|-625257.45|-3447447963466.885254|true|opthamology|victor ellison|2036-07-06 17:42:38|2023-05-13 17:55:44.745449578|2082-09-17
    +69|-16570|-1870912732|-7199983995864711168|-11501.29|-2355822.95|4784415058876.519531|false|american history|holly hernandez|2074-11-26 21:56:49|2076-02-03 03:26:47.100157028|1999-06-03
    +-12|14213|||46313.84|-3912157.22|2214476493452.331055|true|linguistics|luke allen|2052-11-30 04:15:17|2068-10-27 11:57:03.945913098|2006-05-04
    +56||658008867|7868367829080506368|-34336.86|-467709.59|2729407234427.085938|false|undecided|alice hernandez|2058-12-23 05:19:29|2038-03-11 15:26:58.124711805|2064-06-26
    +-30|31795|217476429|2001||-140021.64|2811318250004.752930|false|chemistry|jessica thompson|2040-02-02 16:00:13|2035-08-19 22:22:12.933201371|2082-12-04
    +96|9264|-1057522129|-7429331808102899712|10703.15|-1686203.10|-363688684224.463867|true|kindergarten|ethan quirinius|2081-02-04 05:16:49||
    +-70|23063|-1511162508|8928133990107881472|22991.39|-2823357.23|-786567683392.114258|false|debate|calvin van buren|2054-10-09 00:55:52|2053-06-14 18:30:38.792560125|2077-10-29
    +106|2326|2080412555|-9175279464813223936||3763969.37|-3478279974550.986328||forestry|fred van buren|2079-08-05 12:51:53|2025-05-02 08:19:30.267343837|2007-10-02
    +58|4905|596802082|3467|48922.94|-2427646.65|-1956656626084.800781|true|religion|david brown|2059-08-27 02:37:38|2074-02-26 19:05:09.130777914|2104-03-09
    +67|-31596||2692|23665.84|-2517144.39|-3266719980073.062500|false|debate|nick xylophone|2018-03-26 01:09:51|2048-01-16 05:58:57.246574643|2053-10-18
    +-24|-13948|1063524922|383|-23278.06|961810.86|3461107192485.286133|true|opthamology||2063-12-10 23:49:15|2014-04-14 19:52:11.36942801|2023-10-23
    +-51|-19413|-120704505|2180|7673.97|3840971.12|1717287804088.129883|false|philosophy|luke king|2021-11-07 16:27:24||1974-07-19
    +8|-12564|172075892|7299197687217856512|-43842.34|-3395074.10|-4225250387427.301758|true|forestry|sarah nixon|2053-04-02 00:45:22|2071-09-12 07:22:30.450556241|
    +-107|19917|-1945738830|8295110846998233088|-28431.96|-4322868.81|786529992285.522461

    <TRUNCATED>
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10925: Non-static threadlocals in metastore code can potentially cause memory leak (Vaibhav Gumashta reviewed by Sushanth Sowmyan, Alan Gates, Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c9b7d253
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c9b7d253
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c9b7d253

    Branch: refs/heads/hbase-metastore
    Commit: c9b7d253853573339f54536a552d2564a7a2eadb
    Parents: bef5266
    Author: Vaibhav Gumashta <vgumashta@apache.org>
    Authored: Thu Jun 4 16:20:00 2015 -0700
    Committer: Vaibhav Gumashta <vgumashta@apache.org>
    Committed: Thu Jun 4 16:20:00 2015 -0700

    ----------------------------------------------------------------------
      .../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 4 ++--
      .../authorization/AuthorizationPreEventListener.java | 8 ++++----
      2 files changed, 6 insertions(+), 6 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/c9b7d253/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    index 1688920..85a734c 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    @@ -299,7 +299,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
                }
              };

    - private final ThreadLocal<TxnHandler> threadLocalTxn = new ThreadLocal<TxnHandler>() {
    + private static final ThreadLocal<TxnHandler> threadLocalTxn = new ThreadLocal<TxnHandler>() {
            @Override
            protected synchronized TxnHandler initialValue() {
              return null;
    @@ -316,7 +316,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {

          // Thread local configuration is needed as many threads could make changes
          // to the conf using the connection hook
    - private final ThreadLocal<Configuration> threadLocalConf =
    + private static final ThreadLocal<Configuration> threadLocalConf =
              new ThreadLocal<Configuration>() {
                @Override
                protected synchronized Configuration initialValue() {

    http://git-wip-us.apache.org/repos/asf/hive/blob/c9b7d253/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
    index 63ba327..84f3f76 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
    @@ -68,14 +68,14 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener {
        public static final Log LOG = LogFactory.getLog(
            AuthorizationPreEventListener.class);

    - private final ThreadLocal<Configuration> tConfig = new ThreadLocal<Configuration>() {
    + private static final ThreadLocal<Configuration> tConfig = new ThreadLocal<Configuration>() {
          @Override
          protected Configuration initialValue() {
            return new HiveConf(AuthorizationPreEventListener.class);
          }
        };

    - private final ThreadLocal<HiveMetastoreAuthenticationProvider> tAuthenticator
    + private static final ThreadLocal<HiveMetastoreAuthenticationProvider> tAuthenticator
            = new ThreadLocal<HiveMetastoreAuthenticationProvider>() {
          @Override
          protected HiveMetastoreAuthenticationProvider initialValue() {
    @@ -88,7 +88,7 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener {
          }
        };

    - private final ThreadLocal<List<HiveMetastoreAuthorizationProvider>> tAuthorizers
    + private static final ThreadLocal<List<HiveMetastoreAuthorizationProvider>> tAuthorizers
            = new ThreadLocal<List<HiveMetastoreAuthorizationProvider>>() {
          @Override
          protected List<HiveMetastoreAuthorizationProvider> initialValue() {
    @@ -101,7 +101,7 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener {
          }
        };

    - private final ThreadLocal<Boolean> tConfigSetOnAuths = new ThreadLocal<Boolean>() {
    + private static final ThreadLocal<Boolean> tConfigSetOnAuths = new ThreadLocal<Boolean>() {
          @Override
          protected Boolean initialValue() {
            return false;
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10922: In HS2 doAs=false mode, file system related errors in one query causes other failures (Thejas M Nair, reviewed by Vikram Dixit K)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0e5d907e
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0e5d907e
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0e5d907e

    Branch: refs/heads/hbase-metastore
    Commit: 0e5d907e57304a67752fe68108483c305f6f66a5
    Parents: a96fbde
    Author: Gunther Hagleitner <gunther@apache.org>
    Authored: Thu Jun 4 00:52:47 2015 -0700
    Committer: Gunther Hagleitner <gunther@apache.org>
    Committed: Thu Jun 4 00:55:24 2015 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/metastore/HiveAlterHandler.java | 10 ++++++----
      .../apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java | 1 -
      .../java/org/apache/hadoop/hive/metastore/Warehouse.java | 2 --
      3 files changed, 6 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/0e5d907e/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    index 5391171..99d6680 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    @@ -59,20 +59,24 @@ public class HiveAlterHandler implements AlterHandler {
        private static final Log LOG = LogFactory.getLog(HiveAlterHandler.class
            .getName());

    + @Override
        public Configuration getConf() {
          return hiveConf;
        }

    + @Override
        @SuppressWarnings("nls")
        public void setConf(Configuration conf) {
          hiveConf = conf;
        }

    + @Override
        public void alterTable(RawStore msdb, Warehouse wh, String dbname,
            String name, Table newt) throws InvalidOperationException, MetaException {
          alterTable(msdb, wh, dbname, name, newt, false);
        }

    + @Override
        public void alterTable(RawStore msdb, Warehouse wh, String dbname,
            String name, Table newt, boolean cascade) throws InvalidOperationException, MetaException {
          if (newt == null) {
    @@ -200,8 +204,6 @@ public class HiveAlterHandler implements AlterHandler {
                      + " already exists : " + destPath);
                }
              } catch (IOException e) {
    - Warehouse.closeFs(srcFs);
    - Warehouse.closeFs(destFs);
                throw new InvalidOperationException("Unable to access new location "
                    + destPath + " for table " + newt.getDbName() + "."
                    + newt.getTableName());
    @@ -316,6 +318,7 @@ public class HiveAlterHandler implements AlterHandler {
          }
          return ex.getMessage();
        }
    + @Override
        public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
            final String name, final List<String> part_vals, final Partition new_part)
            throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
    @@ -430,8 +433,6 @@ public class HiveAlterHandler implements AlterHandler {
                      + " already exists : " + destPath);
                  }
                } catch (IOException e) {
    - Warehouse.closeFs(srcFs);
    - Warehouse.closeFs(destFs);
                  throw new InvalidOperationException("Unable to access new location "
                    + destPath + " for partition " + tbl.getDbName() + "."
                    + tbl.getTableName() + " " + new_part.getValues());
    @@ -491,6 +492,7 @@ public class HiveAlterHandler implements AlterHandler {
          return oldPart;
        }

    + @Override
        public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname,
            final String name, final List<Partition> new_parts)
            throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,

    http://git-wip-us.apache.org/repos/asf/hive/blob/0e5d907e/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
    index c3598db..b255090 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
    @@ -60,7 +60,6 @@ public class HiveMetaStoreFsImpl implements MetaStoreFS {
          } catch (FileNotFoundException e) {
            return true; // ok even if there is not data
          } catch (Exception e) {
    - Warehouse.closeFs(fs);
            MetaStoreUtils.logAndThrowMetaException(e);
          }
          return false;

    http://git-wip-us.apache.org/repos/asf/hive/blob/0e5d907e/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    index 011d4d8..bc0f6e3 100755
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    @@ -200,7 +200,6 @@ public class Warehouse {
            fs = getFs(f);
            return FileUtils.mkdir(fs, f, inheritPerms, conf);
          } catch (IOException e) {
    - closeFs(fs);
            MetaStoreUtils.logAndThrowMetaException(e);
          }
          return false;
    @@ -480,7 +479,6 @@ public class Warehouse {
          } catch (FileNotFoundException e) {
            return false;
          } catch (IOException e) {
    - closeFs(fs);
            MetaStoreUtils.logAndThrowMetaException(e);
          }
          return true;
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10427 - collect_list() and collect_set() should accept struct types as argument (Chao Sun, reviewed by Alexander Pivovarov)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c45f925
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c45f925
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c45f925

    Branch: refs/heads/hbase-metastore
    Commit: 9c45f925ebac0c16b1af71aa774cbfce2e65c95a
    Parents: 244386a
    Author: Chao Sun <sunchao@apache.org>
    Authored: Thu Jun 4 13:54:59 2015 -0700
    Committer: Chao Sun <sunchao@apache.org>
    Committed: Thu Jun 4 13:54:59 2015 -0700

    ----------------------------------------------------------------------
      data/files/customers.txt | 3 +
      data/files/nested_orders.txt | 8 +
      data/files/orders.txt | 8 +
      .../ql/udf/generic/GenericUDAFCollectList.java | 15 +-
      .../ql/udf/generic/GenericUDAFCollectSet.java | 14 +-
      .../GenericUDAFMkCollectionEvaluator.java | 21 +-
      .../udaf_collect_set_unsupported.q | 3 +
      .../queries/clientpositive/udaf_collect_set_2.q | 222 ++++++
      .../udaf_collect_set_unsupported.q.out | 1 +
      .../clientpositive/udaf_collect_set_2.q.out | 742 +++++++++++++++++++
      10 files changed, 1017 insertions(+), 20 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/data/files/customers.txt
    ----------------------------------------------------------------------
    diff --git a/data/files/customers.txt b/data/files/customers.txt
    new file mode 100644
    index 0000000..fb85554
    --- /dev/null
    +++ b/data/files/customers.txt
    @@ -0,0 +1,3 @@
    +1,Chris,25
    +2,John,20
    +3,Martin,30

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/data/files/nested_orders.txt
    ----------------------------------------------------------------------
    diff --git a/data/files/nested_orders.txt b/data/files/nested_orders.txt
    new file mode 100644
    index 0000000..b0581dc
    --- /dev/null
    +++ b/data/files/nested_orders.txt
    @@ -0,0 +1,8 @@
    +1,3,2014-05-11,"apple":30.50$"orange":41.35
    +2,1,2013-06-21,"juice":21.45$"bread":15.20
    +3,2,2013-08-10,"yogurt":126.57$"beef":210.57
    +4,1,2014-10-11,"rice":29.36$"grape":1200.50
    +5,3,2014-12-12,"icecream":210.03$"banana":100.56$"coffee:500.00
    +6,2,2015-01-15,"milk":27.45
    +7,2,2014-06-25,"chocolate":3.65$"water":420.36
    +8,1,2013-06-21,"juice":21.45$"bread":15.20

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/data/files/orders.txt
    ----------------------------------------------------------------------
    diff --git a/data/files/orders.txt b/data/files/orders.txt
    new file mode 100644
    index 0000000..06b422b
    --- /dev/null
    +++ b/data/files/orders.txt
    @@ -0,0 +1,8 @@
    +1,3,2014-05-11,30.50
    +2,1,2013-06-21,21.45
    +3,2,2013-08-10,126.57
    +4,1,2014-10-11,29.36
    +5,3,2014-12-12,210.03
    +6,2,2015-01-15,27.45
    +7,2,2014-06-25,3.65
    +8,1,2013-06-21,21.45

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
    index 536c4a7..b10c4ab 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
    @@ -42,10 +42,17 @@ public class GenericUDAFCollectList extends AbstractGenericUDAFResolver {
            throw new UDFArgumentTypeException(parameters.length - 1,
                "Exactly one argument is expected.");
          }
    - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    - throw new UDFArgumentTypeException(0,
    - "Only primitive type arguments are accepted but "
    - + parameters[0].getTypeName() + " was passed as parameter 1.");
    +
    + switch (parameters[0].getCategory()) {
    + case PRIMITIVE:
    + case STRUCT:
    + case MAP:
    + case LIST:
    + break;
    + default:
    + throw new UDFArgumentTypeException(0,
    + "Only primitive, struct, list or map type arguments are accepted but "
    + + parameters[0].getTypeName() + " was passed as parameter 1.");
          }
          return new GenericUDAFMkCollectionEvaluator(BufferType.LIST);
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
    index 6dc424a..312a698 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
    @@ -44,10 +44,16 @@ public class GenericUDAFCollectSet extends AbstractGenericUDAFResolver {
            throw new UDFArgumentTypeException(parameters.length - 1,
                "Exactly one argument is expected.");
          }
    - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    - throw new UDFArgumentTypeException(0,
    - "Only primitive type arguments are accepted but "
    - + parameters[0].getTypeName() + " was passed as parameter 1.");
    + switch (parameters[0].getCategory()) {
    + case PRIMITIVE:
    + case STRUCT:
    + case MAP:
    + case LIST:
    + break;
    + default:
    + throw new UDFArgumentTypeException(0,
    + "Only primitive, struct, list or map type arguments are accepted but "
    + + parameters[0].getTypeName() + " was passed as parameter 1.");
          }
          return new GenericUDAFMkCollectionEvaluator(BufferType.SET);
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
    index efcc8f5..2b5e6dd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
    @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
    -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;

      public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
    @@ -40,7 +39,7 @@ public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
        enum BufferType { SET, LIST }

        // For PARTIAL1 and COMPLETE: ObjectInspectors for original data
    - private transient PrimitiveObjectInspector inputOI;
    + private transient ObjectInspector inputOI;
        // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations (list
        // of objs)
        private transient StandardListObjectInspector loi;
    @@ -64,21 +63,19 @@ public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
          // init output object inspectors
          // The output of a partial aggregation is a list
          if (m == Mode.PARTIAL1) {
    - inputOI = (PrimitiveObjectInspector) parameters[0];
    - return ObjectInspectorFactory
    - .getStandardListObjectInspector((PrimitiveObjectInspector) ObjectInspectorUtils
    - .getStandardObjectInspector(inputOI));
    + inputOI = parameters[0];
    + return ObjectInspectorFactory.getStandardListObjectInspector(
    + ObjectInspectorUtils.getStandardObjectInspector(inputOI));
          } else {
            if (!(parameters[0] instanceof ListObjectInspector)) {
              //no map aggregation.
    - inputOI = (PrimitiveObjectInspector) ObjectInspectorUtils
    - .getStandardObjectInspector(parameters[0]);
    - return (StandardListObjectInspector) ObjectInspectorFactory
    - .getStandardListObjectInspector(inputOI);
    + inputOI = ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
    + return ObjectInspectorFactory.getStandardListObjectInspector(inputOI);
            } else {
              internalMergeOI = (ListObjectInspector) parameters[0];
    - inputOI = (PrimitiveObjectInspector) internalMergeOI.getListElementObjectInspector();
    - loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
    + inputOI = internalMergeOI.getListElementObjectInspector();
    + loi = (StandardListObjectInspector)
    + ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
              return loi;
            }
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q b/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q
    new file mode 100644
    index 0000000..ed03620
    --- /dev/null
    +++ b/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q
    @@ -0,0 +1,3 @@
    +SELECT key, collect_set(create_union(value))
    +FROM src
    +GROUP BY key ORDER BY key limit 20;

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/queries/clientpositive/udaf_collect_set_2.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/udaf_collect_set_2.q b/ql/src/test/queries/clientpositive/udaf_collect_set_2.q
    new file mode 100644
    index 0000000..d18509b
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/udaf_collect_set_2.q
    @@ -0,0 +1,222 @@
    +set hive.support.sql11.reserved.keywords=false;
    +
    +DESCRIBE FUNCTION collect_set;
    +DESCRIBE FUNCTION EXTENDED collect_set;
    +
    +DESCRIBE FUNCTION collect_list;
    +DESCRIBE FUNCTION EXTENDED collect_list;
    +
    +
    +-- initialize tables
    +
    +CREATE TABLE customers (id int, name varchar(10), age int)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ',';
    +
    +LOAD DATA LOCAL INPATH "../../data/files/customers.txt" INTO TABLE customers;
    +
    +CREATE TABLE orders (id int, cid int, date date, amount double)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ',';
    +
    +LOAD DATA LOCAL INPATH "../../data/files/orders.txt" INTO TABLE orders;
    +
    +CREATE TABLE nested_orders (id int, cid int, date date, sub map<string,double>)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +COLLECTION ITEMS TERMINATED BY '$'
    +MAP KEYS TERMINATED BY ':';
    +
    +LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_orders;
    +
    +-- 1. test struct
    +
    +-- 1.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +
    +SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +
    +-- 1.2 when field is map
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +
    +-- 1.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_set(struct(c.name, o.date, map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(struct(c.name, o.date, map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +
    +-- 2. test array
    +
    +-- 2.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(array(o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(array(o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- 2.2 when field is struct
    +
    +SELECT c.id, sort_array(collect_set(array(o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(array(o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- 2.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +
    +-- 3. test map
    +
    +-- 3.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- 3.2 when field is struct
    +
    +SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +-- 3.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id;
    +
    +
    +-- clean up
    +
    +DROP TABLE customer;
    +DROP TABLE orders;
    +DROP TABLE nested_orders

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out b/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out
    new file mode 100644
    index 0000000..4298215
    --- /dev/null
    +++ b/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out
    @@ -0,0 +1 @@
    +FAILED: UDFArgumentTypeException Only primitive, struct, list or map type arguments are accepted but uniontype<> was passed as parameter 1.

    http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out b/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out
    new file mode 100644
    index 0000000..536234f
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out
    @@ -0,0 +1,742 @@
    +PREHOOK: query: DESCRIBE FUNCTION collect_set
    +PREHOOK: type: DESCFUNCTION
    +POSTHOOK: query: DESCRIBE FUNCTION collect_set
    +POSTHOOK: type: DESCFUNCTION
    +collect_set(x) - Returns a set of objects with duplicate elements eliminated
    +PREHOOK: query: DESCRIBE FUNCTION EXTENDED collect_set
    +PREHOOK: type: DESCFUNCTION
    +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED collect_set
    +POSTHOOK: type: DESCFUNCTION
    +collect_set(x) - Returns a set of objects with duplicate elements eliminated
    +PREHOOK: query: DESCRIBE FUNCTION collect_list
    +PREHOOK: type: DESCFUNCTION
    +POSTHOOK: query: DESCRIBE FUNCTION collect_list
    +POSTHOOK: type: DESCFUNCTION
    +collect_list(x) - Returns a list of objects with duplicates
    +PREHOOK: query: DESCRIBE FUNCTION EXTENDED collect_list
    +PREHOOK: type: DESCFUNCTION
    +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED collect_list
    +POSTHOOK: type: DESCFUNCTION
    +collect_list(x) - Returns a list of objects with duplicates
    +PREHOOK: query: -- initialize tables
    +
    +CREATE TABLE customers (id int, name varchar(10), age int)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@customers
    +POSTHOOK: query: -- initialize tables
    +
    +CREATE TABLE customers (id int, name varchar(10), age int)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@customers
    +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/customers.txt" INTO TABLE customers
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@customers
    +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/customers.txt" INTO TABLE customers
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@customers
    +PREHOOK: query: CREATE TABLE orders (id int, cid int, date date, amount double)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@orders
    +POSTHOOK: query: CREATE TABLE orders (id int, cid int, date date, amount double)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@orders
    +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/orders.txt" INTO TABLE orders
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@orders
    +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/orders.txt" INTO TABLE orders
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@orders
    +PREHOOK: query: CREATE TABLE nested_orders (id int, cid int, date date, sub map<string,double>)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +COLLECTION ITEMS TERMINATED BY '$'
    +MAP KEYS TERMINATED BY ':'
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@nested_orders
    +POSTHOOK: query: CREATE TABLE nested_orders (id int, cid int, date date, sub map<string,double>)
    +ROW FORMAT DELIMITED
    +FIELDS TERMINATED BY ','
    +COLLECTION ITEMS TERMINATED BY '$'
    +MAP KEYS TERMINATED BY ':'
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@nested_orders
    +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_orders
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@nested_orders
    +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_orders
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@nested_orders
    +PREHOOK: query: -- 1. test struct
    +
    +-- 1.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 1. test struct
    +
    +-- 1.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","amount":21.45},{"name":"Chris","date":"2014-10-11","amount":29.36}]
    +2 [{"name":"John","date":"2013-08-10","amount":126.57},{"name":"John","date":"2014-06-25","amount":3.65},{"name":"John","date":"2015-01-15","amount":27.45}]
    +3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210.03}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","amount":21.45},{"name":"Chris","date":"2013-06-21","amount":21.45},{"name":"Chris","date":"2014-10-11","amount":29.36}]
    +2 [{"name":"John","date":"2013-08-10","amount":126.57},{"name":"John","date":"2014-06-25","amount":3.65},{"name":"John","date":"2015-01-15","amount":27.45}]
    +3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210.03}]
    +PREHOOK: query: -- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","amount":21.5},{"name":"Chris","date":"2014-10-11","amount":29.4}]
    +2 [{"name":"John","date":"2013-08-10","amount":126.6},{"name":"John","date":"2014-06-25","amount":3.7},{"name":"John","date":"2015-01-15","amount":27.5}]
    +3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","amount":21.5},{"name":"Chris","date":"2013-06-21","amount":21.5},{"name":"Chris","date":"2014-10-11","amount":29.4}]
    +2 [{"name":"John","date":"2013-08-10","amount":126.6},{"name":"John","date":"2014-06-25","amount":3.7},{"name":"John","date":"2015-01-15","amount":27.5}]
    +3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"col1":"Chris","col2":"2013-06-21","col3":21.45},{"col1":"Chris","col2":"2014-10-11","col3":29.36}]
    +2 [{"col1":"John","col2":"2013-08-10","col3":126.57},{"col1":"John","col2":"2014-06-25","col3":3.65},{"col1":"John","col2":"2015-01-15","col3":27.45}]
    +3 [{"col1":"Martin","col2":"2014-05-11","col3":30.5},{"col1":"Martin","col2":"2014-12-12","col3":210.03}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"col1":"Chris","col2":"2013-06-21","col3":21.45},{"col1":"Chris","col2":"2013-06-21","col3":21.45},{"col1":"Chris","col2":"2014-10-11","col3":29.36}]
    +2 [{"col1":"John","col2":"2013-08-10","col3":126.57},{"col1":"John","col2":"2014-06-25","col3":3.65},{"col1":"John","col2":"2015-01-15","col3":27.45}]
    +3 [{"col1":"Martin","col2":"2014-05-11","col3":30.5},{"col1":"Martin","col2":"2014-12-12","col3":210.03}]
    +PREHOOK: query: -- 1.2 when field is map
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 1.2 when field is map
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"name":"Chris","date":"2014-10-11","sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
    +2 [{"name":"John","date":"2013-08-10","sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"name":"John","date":"2014-06-25","sub":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"name":"John","date":"2015-01-15","sub":{"\"milk\"":27.45}}]
    +3 [{"name":"Martin","date":"2014-05-11","sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"name":"Martin","date":"2014-12-12","sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"name":"Chris","date":"2013-06-21","sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"name":"Chris","date":"2014-10-11","sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
    +2 [{"name":"John","date":"2013-08-10","sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"name":"John","date":"2014-06-25","sub":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"name":"John","date":"2015-01-15","sub":{"\"milk\"":27.45}}]
    +3 [{"name":"Martin","date":"2014-05-11","sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"name":"Martin","date":"2014-12-12","sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"col1":"Chris","col2":"2013-06-21","col3":{"\"bread\"":15.2,"\"juice\"":21.45}},{"col1":"Chris","col2":"2014-10-11","col3":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
    +2 [{"col1":"John","col2":"2013-08-10","col3":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"col1":"John","col2":"2014-06-25","col3":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"col1":"John","col2":"2015-01-15","col3":{"\"milk\"":27.45}}]
    +3 [{"col1":"Martin","col2":"2014-05-11","col3":{"\"orange\"":41.35,"\"apple\"":30.5}},{"col1":"Martin","col2":"2014-12-12","col3":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"col1":"Chris","col2":"2013-06-21","col3":{"\"bread\"":15.2,"\"juice\"":21.45}},{"col1":"Chris","col2":"2013-06-21","col3":{"\"bread\"":15.2,"\"juice\"":21.45}},{"col1":"Chris","col2":"2014-10-11","col3":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
    +2 [{"col1":"John","col2":"2013-08-10","col3":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"col1":"John","col2":"2014-06-25","col3":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"col1":"John","col2":"2015-01-15","col3":{"\"milk\"":27.45}}]
    +3 [{"col1":"Martin","col2":"2014-05-11","col3":{"\"orange\"":41.35,"\"apple\"":30.5}},{"col1":"Martin","col2":"2014-12-12","col3":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
    +PREHOOK: query: -- 1.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 1.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","sub":[15.2,21.45]},{"name":"Chris","date":"2014-10-11","sub":[1200.5,29.36]}]
    +2 [{"name":"John","date":"2013-08-10","sub":[210.57,126.57]},{"name":"John","date":"2014-06-25","sub":[3.65,420.36]},{"name":"John","date":"2015-01-15","sub":[27.45]}]
    +3 [{"name":"Martin","date":"2014-05-11","sub":[41.35,30.5]},{"name":"Martin","date":"2014-12-12","sub":[210.03,500.0,100.56]}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"name":"Chris","date":"2013-06-21","sub":[15.2,21.45]},{"name":"Chris","date":"2013-06-21","sub":[15.2,21.45]},{"name":"Chris","date":"2014-10-11","sub":[1200.5,29.36]}]
    +2 [{"name":"John","date":"2013-08-10","sub":[210.57,126.57]},{"name":"John","date":"2014-06-25","sub":[3.65,420.36]},{"name":"John","date":"2015-01-15","sub":[27.45]}]
    +3 [{"name":"Martin","date":"2014-05-11","sub":[41.35,30.5]},{"name":"Martin","date":"2014-12-12","sub":[210.03,500.0,100.56]}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"col1":"Chris","col2":"2013-06-21","col3":[15.2,21.45]},{"col1":"Chris","col2":"2014-10-11","col3":[1200.5,29.36]}]
    +2 [{"col1":"John","col2":"2013-08-10","col3":[210.57,126.57]},{"col1":"John","col2":"2014-06-25","col3":[3.65,420.36]},{"col1":"John","col2":"2015-01-15","col3":[27.45]}]
    +3 [{"col1":"Martin","col2":"2014-05-11","col3":[41.35,30.5]},{"col1":"Martin","col2":"2014-12-12","col3":[210.03,500.0,100.56]}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"col1":"Chris","col2":"2013-06-21","col3":[15.2,21.45]},{"col1":"Chris","col2":"2013-06-21","col3":[15.2,21.45]},{"col1":"Chris","col2":"2014-10-11","col3":[1200.5,29.36]}]
    +2 [{"col1":"John","col2":"2013-08-10","col3":[210.57,126.57]},{"col1":"John","col2":"2014-06-25","col3":[3.65,420.36]},{"col1":"John","col2":"2015-01-15","col3":[27.45]}]
    +3 [{"col1":"Martin","col2":"2014-05-11","col3":[41.35,30.5]},{"col1":"Martin","col2":"2014-12-12","col3":[210.03,500.0,100.56]}]
    +PREHOOK: query: -- 2. test array
    +
    +-- 2.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(array(o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 2. test array
    +
    +-- 2.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(array(o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [[21.45],[29.36]]
    +2 [[3.65],[27.45],[126.57]]
    +3 [[30.5],[210.03]]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(array(o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [[21.45],[21.45],[29.36]]
    +2 [[3.65],[27.45],[126.57]]
    +3 [[30.5],[210.03]]
    +PREHOOK: query: -- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [[21.5],[29.4]]
    +2 [[3.7],[27.5],[126.6]]
    +3 [[30.5],[210]]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [[21.5],[21.5],[29.4]]
    +2 [[3.7],[27.5],[126.6]]
    +3 [[30.5],[210]]
    +PREHOOK: query: -- 2.2 when field is struct
    +
    +SELECT c.id, sort_array(collect_set(array(o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 2.2 when field is struct
    +
    +SELECT c.id, sort_array(collect_set(array(o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [[{"\"bread\"":15.2,"\"juice\"":21.45}],[{"\"grape\"":1200.5,"\"rice\"":29.36}]]
    +2 [[{"\"milk\"":27.45}],[{"\"beef\"":210.57,"\"yogurt\"":126.57}],[{"\"chocolate\"":3.65,"\"water\"":420.36}]]
    +3 [[{"\"orange\"":41.35,"\"apple\"":30.5}],[{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}]]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(array(o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [[{"\"bread\"":15.2,"\"juice\"":21.45}],[{"\"bread\"":15.2,"\"juice\"":21.45}],[{"\"grape\"":1200.5,"\"rice\"":29.36}]]
    +2 [[{"\"milk\"":27.45}],[{"\"beef\"":210.57,"\"yogurt\"":126.57}],[{"\"chocolate\"":3.65,"\"water\"":420.36}]]
    +3 [[{"\"orange\"":41.35,"\"apple\"":30.5}],[{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}]]
    +PREHOOK: query: -- 2.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 2.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [[[15.2,21.45]],[[1200.5,29.36]]]
    +2 [[[3.65,420.36]],[[27.45]],[[210.57,126.57]]]
    +3 [[[41.35,30.5]],[[210.03,500.0,100.56]]]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [[[15.2,21.45]],[[15.2,21.45]],[[1200.5,29.36]]]
    +2 [[[3.65,420.36]],[[27.45]],[[210.57,126.57]]]
    +3 [[[41.35,30.5]],[[210.03,500.0,100.56]]]
    +PREHOOK: query: -- 3. test map
    +
    +-- 3.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 3. test map
    +
    +-- 3.1 when field is primitive
    +
    +SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"amount":21.45},{"amount":29.36}]
    +2 [{"amount":3.65},{"amount":27.45},{"amount":126.57}]
    +3 [{"amount":30.5},{"amount":210.03}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"amount":21.45},{"amount":21.45},{"amount":29.36}]
    +2 [{"amount":3.65},{"amount":27.45},{"amount":126.57}]
    +3 [{"amount":30.5},{"amount":210.03}]
    +PREHOOK: query: -- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- cast decimal
    +
    +SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"amount":21.5},{"amount":29.4}]
    +2 [{"amount":3.7},{"amount":27.5},{"amount":126.6}]
    +3 [{"amount":30.5},{"amount":210}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
    +FROM customers c
    +INNER JOIN orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@orders
    +#### A masked pattern was here ####
    +1 [{"amount":21.5},{"amount":21.5},{"amount":29.4}]
    +2 [{"amount":3.7},{"amount":27.5},{"amount":126.6}]
    +3 [{"amount":30.5},{"amount":210}]
    +PREHOOK: query: -- 3.2 when field is struct
    +
    +SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 3.2 when field is struct
    +
    +SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
    +2 [{"sub":{"\"milk\"":27.45}},{"sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"sub":{"\"chocolate\"":3.65,"\"water\"":420.36}}]
    +3 [{"sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
    +2 [{"sub":{"\"milk\"":27.45}},{"sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"sub":{"\"chocolate\"":3.65,"\"water\"":420.36}}]
    +3 [{"sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
    +PREHOOK: query: -- 3.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 3.3 when field is list
    +
    +SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"sub":[15.2,21.45]},{"sub":[1200.5,29.36]}]
    +2 [{"sub":[3.65,420.36]},{"sub":[27.45]},{"sub":[210.57,126.57]}]
    +3 [{"sub":[41.35,30.5]},{"sub":[210.03,500.0,100.56]}]
    +PREHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@customers
    +PREHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
    +FROM customers c
    +INNER JOIN nested_orders o
    +ON (c.id = o.cid) GROUP BY c.id
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@customers
    +POSTHOOK: Input: default@nested_orders
    +#### A masked pattern was here ####
    +1 [{"sub":[15.2,21.45]},{"sub":[15.2,21.45]},{"sub":[1200.5,29.36]}]
    +2 [{"sub":[3.65,420.36]},{"sub":[27.45]},{"sub":[210.57,126.57]}]
    +3 [{"sub":[41.35,30.5]},{"sub":[210.03,500.0,100.56]}]
    +PREHOOK: query: -- clean up
    +
    +DROP TABLE customer
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: -- clean up
    +
    +DROP TABLE customer
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: DROP TABLE orders
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@orders
    +PREHOOK: Output: default@orders
    +POSTHOOK: query: DROP TABLE orders
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@orders
    +POSTHOOK: Output: default@orders
    +PREHOOK: query: DROP TABLE nested_orders
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@nested_orders
    +PREHOOK: Output: default@nested_orders
    +POSTHOOK: query: DROP TABLE nested_orders
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@nested_orders
    +POSTHOOK: Output: default@nested_orders
  • Gates at Jun 5, 2015 at 11:16 pm
    Merge branch 'master' into hbase-metastore


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a64664a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a64664a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a64664a

    Branch: refs/heads/hbase-metastore
    Commit: 1a64664aedbf0b36544380005b19db79e5c4364c
    Parents: afe1b5e 8a10730
    Author: Alan Gates <gates@hortonworks.com>
    Authored: Fri Jun 5 16:07:58 2015 -0700
    Committer: Alan Gates <gates@hortonworks.com>
    Committed: Fri Jun 5 16:07:58 2015 -0700

    ----------------------------------------------------------------------
      .reviewboardrc | 2 +-
      README.txt | 36 +-
      RELEASE_NOTES.txt | 888 +-
      accumulo-handler/pom.xml | 2 +-
      .../src/test/templates/TestAccumuloCliDriver.vm | 19 +-
      ant/pom.xml | 2 +-
      .../apache/hadoop/hive/ant/GenVectorCode.java | 2 +
      .../apache/hadoop/hive/ant/QTestGenTask.java | 48 +-
      beeline/pom.xml | 8 +-
      .../java/org/apache/hive/beeline/BeeLine.java | 12 +-
      .../org/apache/hive/beeline/BeeLineOpts.java | 9 +
      .../java/org/apache/hive/beeline/Commands.java | 18 +-
      .../apache/hive/beeline/HiveSchemaHelper.java | 1 -
      .../org/apache/hive/beeline/HiveSchemaTool.java | 3 +
      .../hive/beeline/TestBeelineArgParsing.java | 32 +-
      .../test/resources/DummyDriver-1.0-SNAPSHOT.jar | Bin 2812 -> 0 bytes
      beeline/src/test/resources/DummyDriver.txt | 59 +
      .../src/test/resources/postgresql-9.3.jdbc3.jar | Bin 539003 -> 0 bytes
      bin/beeline.cmd | 11 +-
      bin/ext/hiveserver2.cmd | 2 +-
      bin/ext/orcfiledump.sh | 9 +-
      bin/hive | 9 +-
      bin/hive.cmd | 2 +-
      cli/pom.xml | 2 +-
      .../org/apache/hadoop/hive/cli/CliDriver.java | 2 -
      common/pom.xml | 22 +-
      .../apache/hadoop/hive/common/JavaUtils.java | 8 +
      .../hadoop/hive/common/JvmPauseMonitor.java | 225 +
      .../hadoop/hive/common/StatsSetupConst.java | 5 +
      .../common/jsonexplain/tez/TezJsonParser.java | 2 +-
      .../hive/common/jsonexplain/tez/Vertex.java | 2 +-
      .../hive/common/metrics/LegacyMetrics.java | 262 +
      .../hadoop/hive/common/metrics/Metrics.java | 253 -
      .../hive/common/metrics/common/Metrics.java | 68 +
      .../common/metrics/common/MetricsFactory.java | 48 +
      .../metrics/metrics2/CodahaleMetrics.java | 366 +
      .../metrics/metrics2/MetricsReporting.java | 27 +
      .../hadoop/hive/common/type/HiveDecimal.java | 19 +-
      .../org/apache/hadoop/hive/conf/HiveConf.java | 203 +-
      .../apache/hive/common/util/BloomFilter.java | 291 +
      .../apache/hive/common/util/HiveTestUtils.java | 46 +
      .../org/apache/hive/common/util/Murmur3.java | 334 +
      .../apache/hive/common/util/ReflectionUtil.java | 118 +
      .../hive/common/metrics/TestLegacyMetrics.java | 295 +
      .../hadoop/hive/common/metrics/TestMetrics.java | 286 -
      .../metrics/metrics2/TestCodahaleMetrics.java | 138 +
      .../hive/common/type/TestHiveBaseChar.java | 2 +-
      .../hive/common/type/TestHiveVarchar.java | 22 +-
      .../hive/common/util/TestBloomFilter.java | 458 +
      .../apache/hive/common/util/TestMurmur3.java | 189 +
      contrib/pom.xml | 2 +-
      .../test/results/clientpositive/url_hook.q.out | 4 +
      data/files/customers.txt | 3 +
      data/files/nested_orders.txt | 8 +
      data/files/orders.txt | 8 +
      data/files/store_200 | 12 +
      data/files/tjoin1.txt | 3 +
      data/files/tjoin2.txt | 4 +
      data/files/vectortab_a_1k | 1000 ++
      data/files/vectortab_b_1k | 1000 ++
      data/scripts/q_test_cleanup_for_encryption.sql | 2 +-
      data/scripts/q_test_init_for_encryption.sql | 2 +-
      dev-support/jenkins-execute-build.sh | 6 +-
      dev-support/jenkins-execute-hms-test.sh | 26 +-
      dev-support/jenkins-submit-build.sh | 4 +
      hbase-handler/pom.xml | 2 +-
      .../hadoop/hive/hbase/HBaseStorageHandler.java | 8 +
      .../hive/hbase/HBaseTestStructSerializer.java | 20 +-
      .../src/test/queries/positive/hbase_timestamp.q | 22 +-
      .../results/positive/external_table_ppd.q.out | 8 +-
      .../results/positive/hbase_custom_key2.q.out | 18 +-
      .../results/positive/hbase_custom_key3.q.out | 24 +-
      .../results/positive/hbase_ppd_key_range.q.out | 76 +-
      .../test/results/positive/hbase_pushdown.q.out | 68 +-
      .../test/results/positive/hbase_queries.q.out | 42 +-
      .../test/results/positive/hbase_timestamp.q.out | 92 +-
      .../test/results/positive/ppd_key_ranges.q.out | 32 +-
      .../src/test/templates/TestHBaseCliDriver.vm | 18 +-
      .../templates/TestHBaseNegativeCliDriver.vm | 19 +-
      hcatalog/core/pom.xml | 2 +-
      .../hive/hcatalog/common/HCatException.java | 2 +-
      .../apache/hive/hcatalog/common/HCatUtil.java | 37 +-
      .../hive/hcatalog/common/HiveClientCache.java | 94 +-
      .../data/HCatRecordObjectInspectorFactory.java | 2 +-
      .../hive/hcatalog/data/HCatRecordSerDe.java | 9 +-
      .../apache/hive/hcatalog/data/JsonSerDe.java | 7 +-
      .../hcatalog/data/schema/HCatFieldSchema.java | 6 +-
      .../DefaultOutputCommitterContainer.java | 6 +-
      .../mapreduce/FileOutputCommitterContainer.java | 22 +-
      .../mapreduce/FileOutputFormatContainer.java | 8 +-
      .../hcatalog/mapreduce/HCatBaseInputFormat.java | 50 +-
      .../hcatalog/mapreduce/HCatOutputFormat.java | 6 +-
      .../hcatalog/mapreduce/HCatRecordReader.java | 4 +-
      .../hive/hcatalog/mapreduce/HCatSplit.java | 21 +-
      .../hive/hcatalog/mapreduce/HCatTableInfo.java | 12 +
      .../hcatalog/mapreduce/InitializeInput.java | 6 +-
      .../hive/hcatalog/mapreduce/InputJobInfo.java | 5 +
      .../hive/hcatalog/mapreduce/InternalUtil.java | 2 +
      .../hive/hcatalog/mapreduce/OutputJobInfo.java | 2 +-
      .../hive/hcatalog/mapreduce/PartInfo.java | 117 +-
      .../hive/hcatalog/mapreduce/Security.java | 10 +-
      .../mapreduce/TaskCommitContextRegistry.java | 6 +-
      .../hcatalog/common/TestHiveClientCache.java | 37 +-
      .../hive/hcatalog/data/TestJsonSerDe.java | 2 +-
      .../hcatalog/mapreduce/HCatMapReduceTest.java | 3 +-
      .../mapreduce/TestHCatOutputFormat.java | 5 +-
      .../hcatalog/mapreduce/TestHCatPartitioned.java | 32 +-
      .../hcatalog/mapreduce/TestPassProperties.java | 5 +-
      hcatalog/hcatalog-pig-adapter/pom.xml | 2 +-
      .../apache/hive/hcatalog/pig/HCatLoader.java | 8 -
      .../apache/hive/hcatalog/pig/PigHCatUtil.java | 10 +-
      .../hive/hcatalog/pig/TestHCatLoader.java | 44 -
      .../hcatalog/pig/TestHCatLoaderEncryption.java | 12 +-
      hcatalog/pom.xml | 2 +-
      hcatalog/server-extensions/pom.xml | 2 +-
      .../listener/DbNotificationListener.java | 6 +-
      .../hcatalog/listener/NotificationListener.java | 23 +-
      .../messaging/AlterPartitionMessage.java | 6 +-
      .../hive/hcatalog/messaging/InsertMessage.java | 7 +-
      .../hive/hcatalog/messaging/MessageFactory.java | 10 +-
      .../json/JSONAlterPartitionMessage.java | 14 +-
      .../messaging/json/JSONAlterTableMessage.java | 5 +
      .../messaging/json/JSONInsertMessage.java | 15 +-
      .../messaging/json/JSONMessageFactory.java | 14 +-
      .../listener/TestNotificationListener.java | 8 +-
      hcatalog/src/test/e2e/templeton/README.txt | 13 +
      hcatalog/src/test/e2e/templeton/build.xml | 29 +
      .../deployers/config/hive/hive-log4j.properties | 88 +
      .../deployers/config/hive/hive-site.mysql.xml | 77 +
      .../webhcat/webhcat-site.updateConfig.xml | 118 +
      .../deployers/config/webhcat/webhcat-site.xml | 9 +-
      .../templeton/deployers/deploy_e2e_artifacts.sh | 6 +
      .../src/test/e2e/templeton/deployers/env.sh | 7 +
      .../deployers/modify_webhcat_config.sh | 40 +
      .../deployers/restore_webhcat_config.sh | 36 +
      .../templeton/deployers/start_hive_services.sh | 7 +
      .../src/test/e2e/templeton/inpdir/xmlmapper.py | 15 +
      .../src/test/e2e/templeton/inpdir/xmlreducer.py | 16 +
      .../tests/jobsubmission_streaming.conf | 2 +-
      .../templeton/tests/modifyConfiguration.conf | 67 +
      hcatalog/streaming/pom.xml | 2 +-
      .../streaming/AbstractRecordWriter.java | 11 +-
      .../streaming/DelimitedInputWriter.java | 2 +-
      .../hive/hcatalog/streaming/HiveEndPoint.java | 11 +-
      .../hive/hcatalog/streaming/TestStreaming.java | 2 +-
      hcatalog/webhcat/java-client/pom.xml | 2 +-
      .../apache/hive/hcatalog/api/HCatClient.java | 18 +
      .../hive/hcatalog/api/HCatClientHMSImpl.java | 22 +-
      .../hcatalog/api/HCatNotificationEvent.java | 2 +-
      .../apache/hive/hcatalog/api/HCatPartition.java | 12 +
      .../api/repl/HCatReplicationTaskIterator.java | 81 +-
      .../hcatalog/api/repl/NoopReplicationTask.java | 9 +-
      .../hive/hcatalog/api/repl/ReplicationTask.java | 103 +-
      .../hcatalog/api/repl/ReplicationUtils.java | 31 +-
      .../api/repl/commands/DropDatabaseCommand.java | 99 +
      .../api/repl/commands/DropPartitionCommand.java | 122 +
      .../api/repl/commands/DropTableCommand.java | 113 +
      .../api/repl/commands/ExportCommand.java | 131 +
      .../api/repl/commands/ImportCommand.java | 139 +
      .../hcatalog/api/repl/commands/NoopCommand.java | 22 +-
      .../repl/exim/AddPartitionReplicationTask.java | 111 +
      .../exim/AlterPartitionReplicationTask.java | 88 +
      .../repl/exim/AlterTableReplicationTask.java | 85 +
      .../exim/CreateDatabaseReplicationTask.java | 38 +
      .../repl/exim/CreateTableReplicationTask.java | 85 +
      .../repl/exim/DropDatabaseReplicationTask.java | 56 +
      .../repl/exim/DropPartitionReplicationTask.java | 77 +
      .../api/repl/exim/DropTableReplicationTask.java | 56 +
      .../repl/exim/EximReplicationTaskFactory.java | 63 +
      .../api/repl/exim/InsertReplicationTask.java | 99 +
      .../hive/hcatalog/api/TestHCatClient.java | 126 +-
      .../hcatalog/api/repl/CommandTestUtils.java | 107 +
      .../hcatalog/api/repl/TestReplicationTask.java | 92 +
      .../api/repl/commands/TestCommands.java | 590 +
      .../api/repl/commands/TestNoopCommand.java | 42 +
      .../api/repl/exim/TestEximReplicationTasks.java | 600 +
      hcatalog/webhcat/svr/pom.xml | 15 +-
      .../svr/src/main/config/webhcat-default.xml | 10 +-
      .../hcatalog/templeton/CompleteDelegator.java | 6 +-
      .../hive/hcatalog/templeton/HcatDelegator.java | 4 +-
      .../hcatalog/templeton/SecureProxySupport.java | 9 +-
      .../templeton/tool/JobSubmissionConstants.java | 1 +
      .../hcatalog/templeton/tool/LaunchMapper.java | 29 +-
      .../templeton/tool/TempletonControllerJob.java | 15 +-
      hwi/pom.xml | 2 +-
      itests/custom-serde/pom.xml | 2 +-
      itests/hcatalog-unit/pom.xml | 2 +-
      .../listener/TestDbNotificationListener.java | 75 +-
      itests/hive-jmh/pom.xml | 2 +-
      .../benchmark/storage/ColumnarStorageBench.java | 4 +-
      .../vectorization/VectorizationBench.java | 146 +-
      itests/hive-minikdc/pom.xml | 2 +-
      .../hive/minikdc/TestHs2HooksWithMiniKdc.java | 2 +-
      .../hive/minikdc/TestJdbcWithMiniKdcCookie.java | 102 +
      itests/hive-unit-hadoop2/pom.xml | 2 +-
      .../apache/hive/jdbc/TestSchedulerQueue.java | 3 +-
      itests/hive-unit/pom.xml | 135 +-
      .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 57 +-
      .../org/apache/hadoop/hive/UtilsForTest.java | 41 +
      .../apache/hadoop/hive/hooks/TestHs2Hooks.java | 10 +-
      .../hadoop/hive/metastore/TestFilterHooks.java | 29 +-
      .../hive/metastore/TestHiveMetaStore.java | 30 +
      .../hadoop/hive/metastore/TestHiveMetaTool.java | 17 +-
      .../metastore/TestMetaStoreEventListener.java | 14 +-
      .../hive/metastore/TestMetaStoreMetrics.java | 94 +
      .../TestHiveAuthorizerCheckInvocation.java | 34 +-
      .../plugin/TestHiveAuthorizerShowFilters.java | 6 +-
      .../hive/beeline/TestBeeLineWithArgs.java | 12 +
      .../org/apache/hive/jdbc/TestJdbcDriver2.java | 20 +
      .../jdbc/TestJdbcWithLocalClusterSpark.java | 3 +-
      .../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 123 +
      .../apache/hive/jdbc/TestJdbcWithMiniMr.java | 3 +-
      ...stMultiSessionsHS2WithLocalClusterSpark.java | 3 +-
      .../authorization/TestJdbcMetadataApiAuth.java | 260 +
      .../TestJdbcWithSQLAuthorization.java | 43 +
      .../operation/OperationLoggingAPITestBase.java | 235 +
      .../cli/operation/TestOperationLoggingAPI.java | 379 -
      .../TestOperationLoggingAPIWithMr.java | 168 +
      .../TestOperationLoggingAPIWithTez.java | 54 +
      .../cli/thrift/TestThriftHttpCLIService.java | 74 +-
      itests/pom.xml | 102 +-
      itests/qtest-spark/pom.xml | 2 +-
      itests/qtest/pom.xml | 60 +-
      .../test/resources/testconfiguration.properties | 68 +-
      itests/test-serde/pom.xml | 2 +-
      itests/util/pom.xml | 2 +-
      .../org/apache/hadoop/hive/ql/QTestUtil.java | 81 +-
      .../generic/GenericUDFTestGetJavaBoolean.java | 8 +-
      jdbc/pom.xml | 2 +-
      .../org/apache/hive/jdbc/HiveConnection.java | 151 +-
      .../apache/hive/jdbc/HivePreparedStatement.java | 2 +-
      .../apache/hive/jdbc/HiveQueryResultSet.java | 50 +-
      .../org/apache/hive/jdbc/HiveStatement.java | 32 +-
      .../hive/jdbc/HttpBasicAuthInterceptor.java | 49 +-
      .../jdbc/HttpKerberosRequestInterceptor.java | 62 +-
      .../hive/jdbc/HttpRequestInterceptorBase.java | 89 +
      jdbc/src/java/org/apache/hive/jdbc/Utils.java | 22 +-
      .../hive/jdbc/ZooKeeperHiveClientHelper.java | 2 +-
      metastore/bin/.gitignore | 3 +-
      metastore/if/hive_metastore.thrift | 14 +-
      metastore/pom.xml | 2 +-
      .../upgrade/derby/hive-schema-1.3.0.derby.sql | 336 +
      .../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 3 +
      .../scripts/upgrade/derby/upgrade.order.derby | 1 +
      .../upgrade/mssql/006-HIVE-9456.mssql.sql | 323 +
      .../upgrade/mssql/hive-schema-1.2.0.mssql.sql | 256 +-
      .../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 947 ++
      .../mssql/upgrade-1.1.0-to-1.2.0.mssql.sql | 1 +
      .../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 5 +
      .../scripts/upgrade/mssql/upgrade.order.mssql | 1 +
      .../upgrade/mysql/021-HIVE-7018.mysql.sql | 53 -
      .../upgrade/mysql/hive-schema-1.2.0.mysql.sql | 10 +-
      .../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 840 +
      .../mysql/upgrade-1.1.0-to-1.2.0.mysql.sql | 2 +-
      .../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 4 +
      .../scripts/upgrade/mysql/upgrade.order.mysql | 1 +
      .../upgrade/oracle/hive-schema-1.3.0.oracle.sql | 788 +
      .../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 4 +
      .../scripts/upgrade/oracle/upgrade.order.oracle | 1 +
      .../postgres/hive-schema-1.3.0.postgres.sql | 1457 ++
      .../upgrade-1.2.0-to-1.3.0.postgres.sql | 12 +
      .../upgrade/postgres/upgrade.order.postgres | 1 +
      .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 214 +-
      .../gen/thrift/gen-cpp/hive_metastore_types.h | 146 +-
      .../metastore/api/AddDynamicPartitions.java | 4 +-
      .../metastore/api/AddPartitionsRequest.java | 4 +-
      .../hive/metastore/api/AddPartitionsResult.java | 4 +-
      .../hadoop/hive/metastore/api/AggrStats.java | 4 +-
      .../hive/metastore/api/ColumnStatistics.java | 4 +-
      .../metastore/api/ColumnStatisticsData.java | 63 +-
      .../apache/hadoop/hive/metastore/api/Date.java | 383 +
      .../hive/metastore/api/DateColumnStatsData.java | 704 +
      .../metastore/api/DropPartitionsResult.java | 4 +-
      .../hadoop/hive/metastore/api/FieldSchema.java | 18 +-
      .../hive/metastore/api/FireEventRequest.java | 4 +-
      .../hadoop/hive/metastore/api/Function.java | 4 +-
      .../metastore/api/GetOpenTxnsInfoResponse.java | 4 +-
      .../hive/metastore/api/GetOpenTxnsResponse.java | 4 +-
      .../api/GetPrincipalsInRoleResponse.java | 4 +-
      .../api/GetRoleGrantsForPrincipalResponse.java | 4 +-
      .../api/HeartbeatTxnRangeResponse.java | 8 +-
      .../hive/metastore/api/HiveObjectRef.java | 4 +-
      .../metastore/api/InsertEventRequestData.java | 4 +-
      .../hadoop/hive/metastore/api/LockRequest.java | 4 +-
      .../api/NotificationEventResponse.java | 4 +-
      .../hive/metastore/api/OpenTxnsResponse.java | 4 +-
      .../hadoop/hive/metastore/api/Partition.java | 24 +-
      .../api/PartitionListComposingSpec.java | 4 +-
      .../api/PartitionSpecWithSharedSD.java | 4 +-
      .../hive/metastore/api/PartitionWithoutSD.java | 4 +-
      .../metastore/api/PartitionsByExprResult.java | 4 +-
      .../metastore/api/PartitionsStatsRequest.java | 8 +-
      .../metastore/api/PartitionsStatsResult.java | 4 +-
      .../metastore/api/PrincipalPrivilegeSet.java | 12 +-
      .../hadoop/hive/metastore/api/PrivilegeBag.java | 4 +-
      .../hive/metastore/api/RequestPartsSpec.java | 8 +-
      .../hadoop/hive/metastore/api/Schema.java | 4 +-
      .../hadoop/hive/metastore/api/SerDeInfo.java | 20 +-
      .../api/SetPartitionsStatsRequest.java | 4 +-
      .../hive/metastore/api/ShowCompactResponse.java | 4 +-
      .../hive/metastore/api/ShowLocksResponse.java | 4 +-
      .../hadoop/hive/metastore/api/SkewedInfo.java | 16 +-
      .../hive/metastore/api/StorageDescriptor.java | 42 +-
      .../apache/hadoop/hive/metastore/api/Table.java | 4 +-
      .../hive/metastore/api/TableStatsRequest.java | 4 +-
      .../hive/metastore/api/TableStatsResult.java | 4 +-
      .../hive/metastore/api/ThriftHiveMetastore.java | 196 +-
      .../apache/hadoop/hive/metastore/api/Type.java | 4 +-
      .../src/gen/thrift/gen-php/metastore/Types.php | 239 +
      .../gen/thrift/gen-py/hive_metastore/ttypes.py | 179 +-
      .../gen/thrift/gen-rb/hive_metastore_types.rb | 49 +-
      .../hive/metastore/AggregateStatsCache.java | 572 +
      .../hadoop/hive/metastore/HiveAlterHandler.java | 42 +-
      .../hadoop/hive/metastore/HiveMetaStore.java | 221 +-
      .../hive/metastore/HiveMetaStoreClient.java | 26 +-
      .../hive/metastore/HiveMetaStoreFsImpl.java | 1 -
      .../hadoop/hive/metastore/IMetaStoreClient.java | 11 +
      .../hive/metastore/MetaStoreDirectSql.java | 81 +-
      .../hadoop/hive/metastore/MetaStoreUtils.java | 27 +-
      .../hadoop/hive/metastore/ObjectStore.java | 107 +-
      .../hive/metastore/RetryingMetaStoreClient.java | 89 +-
      .../hive/metastore/StatObjectConverter.java | 47 +
      .../apache/hadoop/hive/metastore/Warehouse.java | 14 +-
      .../metastore/events/DropPartitionEvent.java | 14 +-
      .../hive/metastore/events/InsertEvent.java | 31 +-
      .../metastore/events/PreDropPartitionEvent.java | 23 +-
      .../hive/metastore/tools/HiveMetaTool.java | 35 +-
      .../metastore/txn/CompactionTxnHandler.java | 9 +-
      .../hadoop/hive/metastore/txn/TxnHandler.java | 221 +-
      .../model/MPartitionColumnStatistics.java | 8 +
      .../metastore/model/MTableColumnStatistics.java | 7 +
      .../hive/metastore/TestAggregateStatsCache.java | 266 +
      .../hive/metastore/TestHiveMetastoreCli.java | 63 +
      .../hive/metastore/txn/TestTxnHandler.java | 40 +-
      odbc/pom.xml | 2 +-
      packaging/pom.xml | 2 +-
      packaging/src/main/assembly/bin.xml | 1 +
      pom.xml | 17 +-
      ql/.gitignore | 2 +
      ql/pom.xml | 18 +-
      .../ExpressionTemplates/ColumnDivideColumn.txt | 26 +-
      .../java/org/apache/hadoop/hive/ql/Context.java | 10 +-
      .../java/org/apache/hadoop/hive/ql/Driver.java | 10 +-
      .../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 +
      .../hadoop/hive/ql/exec/ColumnStatsTask.java | 27 +
      .../hive/ql/exec/ColumnStatsUpdateTask.java | 37 +
      .../hive/ql/exec/CommonMergeJoinOperator.java | 104 +-
      .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 137 +-
      .../hive/ql/exec/DefaultFetchFormatter.java | 4 +-
      .../hadoop/hive/ql/exec/DemuxOperator.java | 6 +-
      .../apache/hadoop/hive/ql/exec/ExplainTask.java | 10 +-
      .../hive/ql/exec/ExprNodeEvaluatorFactory.java | 12 +-
      .../ql/exec/ExprNodeGenericFuncEvaluator.java | 3 +-
      .../hive/ql/exec/ExprNodeNullEvaluator.java | 47 -
      .../hadoop/hive/ql/exec/FetchOperator.java | 15 +-
      .../apache/hadoop/hive/ql/exec/FetchTask.java | 2 +
      .../hadoop/hive/ql/exec/FileSinkOperator.java | 47 +-
      .../hadoop/hive/ql/exec/FunctionRegistry.java | 28 +-
      .../hadoop/hive/ql/exec/HashTableLoader.java | 4 +-
      .../hadoop/hive/ql/exec/JoinOperator.java | 3 +-
      .../apache/hadoop/hive/ql/exec/JoinUtil.java | 4 +-
      .../hadoop/hive/ql/exec/MapJoinOperator.java | 295 +-
      .../apache/hadoop/hive/ql/exec/ObjectCache.java | 7 +
      .../apache/hadoop/hive/ql/exec/Operator.java | 4 +
      .../hadoop/hive/ql/exec/OperatorFactory.java | 22 +-
      .../hive/ql/exec/PTFRollingPartition.java | 30 +-
      .../hadoop/hive/ql/exec/ReduceSinkOperator.java | 3 +-
      .../apache/hadoop/hive/ql/exec/Registry.java | 29 +-
      .../apache/hadoop/hive/ql/exec/RowSchema.java | 9 +
      .../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 4 +-
      .../hadoop/hive/ql/exec/SecureCmdDoAs.java | 5 +-
      .../ql/exec/SparkHashTableSinkOperator.java | 35 +-
      .../hadoop/hive/ql/exec/StatsNoJobTask.java | 6 +-
      .../apache/hadoop/hive/ql/exec/TaskRunner.java | 1 +
      .../apache/hadoop/hive/ql/exec/Utilities.java | 82 +-
      .../hadoop/hive/ql/exec/mr/ExecDriver.java | 5 +-
      .../hadoop/hive/ql/exec/mr/HashTableLoader.java | 2 +-
      .../hadoop/hive/ql/exec/mr/ObjectCache.java | 5 +
      .../persistence/BytesBytesMultiHashMap.java | 49 +-
      .../exec/persistence/HybridHashTableConf.java | 86 +
      .../persistence/HybridHashTableContainer.java | 357 +-
      .../ql/exec/persistence/KeyValueContainer.java | 31 +-
      .../persistence/MapJoinBytesTableContainer.java | 92 +-
      .../hive/ql/exec/persistence/MapJoinKey.java | 14 +
      .../MapJoinTableContainerDirectAccess.java | 31 +
      .../ql/exec/persistence/ObjectContainer.java | 31 +-
      .../ReusableGetAdaptorDirectAccess.java | 30 +
      .../hive/ql/exec/persistence/RowContainer.java | 4 +-
      .../hadoop/hive/ql/exec/spark/CacheTran.java | 54 +
      .../hive/ql/exec/spark/GroupByShuffler.java | 4 +
      .../hive/ql/exec/spark/HashTableLoader.java | 25 +-
      .../ql/exec/spark/HivePairFlatMapFunction.java | 1 +
      .../ql/exec/spark/HiveSparkClientFactory.java | 29 +-
      .../hive/ql/exec/spark/KryoSerializer.java | 4 +-
      .../ql/exec/spark/LocalHiveSparkClient.java | 8 +-
      .../hadoop/hive/ql/exec/spark/MapInput.java | 16 +
      .../hadoop/hive/ql/exec/spark/MapTran.java | 22 +-
      .../hadoop/hive/ql/exec/spark/ReduceTran.java | 23 +-
      .../ql/exec/spark/RemoteHiveSparkClient.java | 36 +-
      .../hadoop/hive/ql/exec/spark/ShuffleTran.java | 24 +
      .../hive/ql/exec/spark/SmallTableCache.java | 73 +
      .../hive/ql/exec/spark/SortByShuffler.java | 5 +
      .../ql/exec/spark/SparkMapRecordHandler.java | 1 +
      .../hadoop/hive/ql/exec/spark/SparkPlan.java | 150 +
      .../hive/ql/exec/spark/SparkPlanGenerator.java | 58 +-
      .../ql/exec/spark/SparkReduceRecordHandler.java | 1 +
      .../hive/ql/exec/spark/SparkShuffler.java | 2 +
      .../hadoop/hive/ql/exec/spark/SparkTask.java | 16 +
      .../hadoop/hive/ql/exec/spark/SparkTran.java | 6 +
      .../hive/ql/exec/spark/SparkUtilities.java | 72 +-
      .../ql/exec/spark/session/SparkSessionImpl.java | 2 +-
      .../spark/session/SparkSessionManagerImpl.java | 63 +-
      .../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
      .../spark/status/impl/JobMetricsListener.java | 12 +
      .../hadoop/hive/ql/exec/tez/DagUtils.java | 1 +
      .../hive/ql/exec/tez/HashTableLoader.java | 83 +-
      .../hadoop/hive/ql/exec/tez/ObjectCache.java | 6 +
      .../hive/ql/exec/tez/ReduceRecordSource.java | 148 +-
      .../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 14 +-
      .../hive/ql/exec/tez/TezSessionState.java | 13 +-
      .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 3 +-
      .../hive/ql/exec/vector/BytesColumnVector.java | 9 +
      .../hive/ql/exec/vector/ColumnVector.java | 10 +
      .../vector/VectorAppMasterEventOperator.java | 123 +-
      .../ql/exec/vector/VectorColumnMapping.java | 6 +-
      .../ql/exec/vector/VectorColumnOrderedMap.java | 17 +-
      .../exec/vector/VectorColumnOutputMapping.java | 4 +
      .../ql/exec/vector/VectorColumnSetInfo.java | 41 +-
      .../exec/vector/VectorColumnSourceMapping.java | 4 +
      .../hive/ql/exec/vector/VectorCopyRow.java | 35 +-
      .../ql/exec/vector/VectorDeserializeRow.java | 13 +-
      .../ql/exec/vector/VectorFileSinkOperator.java | 78 +-
      .../ql/exec/vector/VectorFilterOperator.java | 5 +-
      .../ql/exec/vector/VectorGroupByOperator.java | 26 +-
      .../exec/vector/VectorMapJoinBaseOperator.java | 185 +
      .../ql/exec/vector/VectorMapJoinOperator.java | 171 +-
      .../VectorMapJoinOuterFilteredOperator.java | 122 +
      .../exec/vector/VectorReduceSinkOperator.java | 83 +-
      .../exec/vector/VectorSMBMapJoinOperator.java | 21 +-
      .../hive/ql/exec/vector/VectorSerializeRow.java | 17 +
      .../exec/vector/VectorSerializeRowNoNulls.java | 17 +
      .../ql/exec/vector/VectorizationContext.java | 135 +-
      .../ql/exec/vector/VectorizedBatchUtil.java | 56 +-
      .../ql/exec/vector/VectorizedRowBatchCtx.java | 4 +-
      .../expressions/ConstantVectorExpression.java | 3 -
      .../vector/expressions/IdentityExpression.java | 9 +
      .../ql/exec/vector/expressions/MathExpr.java | 9 +-
      .../VectorExpressionWriterFactory.java | 29 +-
      .../mapjoin/VectorMapJoinCommonOperator.java | 795 +
      .../VectorMapJoinGenerateResultOperator.java | 852 +
      ...pJoinInnerBigOnlyGenerateResultOperator.java | 328 +
      .../VectorMapJoinInnerBigOnlyLongOperator.java | 374 +
      ...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 387 +
      ...VectorMapJoinInnerBigOnlyStringOperator.java | 366 +
      ...ectorMapJoinInnerGenerateResultOperator.java | 228 +
      .../mapjoin/VectorMapJoinInnerLongOperator.java | 371 +
      .../VectorMapJoinInnerMultiKeyOperator.java | 383 +
      .../VectorMapJoinInnerStringOperator.java | 360 +
      ...orMapJoinLeftSemiGenerateResultOperator.java | 225 +
      .../VectorMapJoinLeftSemiLongOperator.java | 364 +
      .../VectorMapJoinLeftSemiMultiKeyOperator.java | 380 +
      .../VectorMapJoinLeftSemiStringOperator.java | 353 +
      ...ectorMapJoinOuterGenerateResultOperator.java | 745 +
      .../mapjoin/VectorMapJoinOuterLongOperator.java | 447 +
      .../VectorMapJoinOuterMultiKeyOperator.java | 464 +
      .../VectorMapJoinOuterStringOperator.java | 433 +
      .../mapjoin/VectorMapJoinRowBytesContainer.java | 318 +
      .../fast/VectorMapJoinFastBytesHashMap.java | 101 +
      .../VectorMapJoinFastBytesHashMultiSet.java | 93 +
      .../fast/VectorMapJoinFastBytesHashSet.java | 85 +
      .../fast/VectorMapJoinFastBytesHashTable.java | 221 +
      .../fast/VectorMapJoinFastBytesHashUtil.java | 41 +
      .../mapjoin/fast/VectorMapJoinFastHashMap.java | 38 +
      .../fast/VectorMapJoinFastHashMultiSet.java | 48 +
      .../mapjoin/fast/VectorMapJoinFastHashSet.java | 44 +
      .../fast/VectorMapJoinFastHashTable.java | 68 +
      .../fast/VectorMapJoinFastHashTableLoader.java | 114 +
      .../fast/VectorMapJoinFastIntHashUtil.java | 32 +
      .../mapjoin/fast/VectorMapJoinFastKeyStore.java | 173 +
      .../fast/VectorMapJoinFastLongHashMap.java | 94 +
      .../fast/VectorMapJoinFastLongHashMultiSet.java | 91 +
      .../fast/VectorMapJoinFastLongHashSet.java | 84 +
      .../fast/VectorMapJoinFastLongHashTable.java | 284 +
      .../fast/VectorMapJoinFastLongHashUtil.java | 63 +
      .../fast/VectorMapJoinFastMultiKeyHashMap.java | 39 +
      .../VectorMapJoinFastMultiKeyHashMultiSet.java | 32 +
      .../fast/VectorMapJoinFastMultiKeyHashSet.java | 32 +
      .../fast/VectorMapJoinFastStringCommon.java | 67 +
      .../fast/VectorMapJoinFastStringHashMap.java | 44 +
      .../VectorMapJoinFastStringHashMultiSet.java | 44 +
      .../fast/VectorMapJoinFastStringHashSet.java | 44 +
      .../fast/VectorMapJoinFastTableContainer.java | 222 +
      .../fast/VectorMapJoinFastValueStore.java | 557 +
      .../hashtable/VectorMapJoinBytesHashMap.java | 51 +
      .../VectorMapJoinBytesHashMultiSet.java | 51 +
      .../hashtable/VectorMapJoinBytesHashSet.java | 51 +
      .../hashtable/VectorMapJoinBytesHashTable.java | 26 +
      .../mapjoin/hashtable/VectorMapJoinHashMap.java | 34 +
      .../hashtable/VectorMapJoinHashMapResult.java | 63 +
      .../hashtable/VectorMapJoinHashMultiSet.java | 31 +
      .../VectorMapJoinHashMultiSetResult.java | 34 +
      .../mapjoin/hashtable/VectorMapJoinHashSet.java | 34 +
      .../hashtable/VectorMapJoinHashSetResult.java | 28 +
      .../hashtable/VectorMapJoinHashTable.java | 43 +
      .../hashtable/VectorMapJoinHashTableResult.java | 81 +
      .../hashtable/VectorMapJoinLongHashMap.java | 46 +
      .../VectorMapJoinLongHashMultiSet.java | 46 +
      .../hashtable/VectorMapJoinLongHashSet.java | 46 +
      .../hashtable/VectorMapJoinLongHashTable.java | 31 +
      .../hashtable/VectorMapJoinTableContainer.java | 28 +
      .../VectorMapJoinOptimizedCreateHashTable.java | 129 +
      .../VectorMapJoinOptimizedHashMap.java | 128 +
      .../VectorMapJoinOptimizedHashMultiSet.java | 103 +
      .../VectorMapJoinOptimizedHashSet.java | 78 +
      .../VectorMapJoinOptimizedHashTable.java | 95 +
      .../VectorMapJoinOptimizedLongCommon.java | 171 +
      .../VectorMapJoinOptimizedLongHashMap.java | 82 +
      .../VectorMapJoinOptimizedLongHashMultiSet.java | 83 +
      .../VectorMapJoinOptimizedLongHashSet.java | 83 +
      .../VectorMapJoinOptimizedMultiKeyHashMap.java | 36 +
      ...torMapJoinOptimizedMultiKeyHashMultiSet.java | 36 +
      .../VectorMapJoinOptimizedMultiKeyHashSet.java | 36 +
      .../VectorMapJoinOptimizedStringCommon.java | 98 +
      .../VectorMapJoinOptimizedStringHashMap.java | 63 +
      ...ectorMapJoinOptimizedStringHashMultiSet.java | 64 +
      .../VectorMapJoinOptimizedStringHashSet.java | 63 +
      .../hadoop/hive/ql/history/HiveHistory.java | 3 +-
      .../hive/ql/io/BucketizedHiveInputSplit.java | 5 +-
      .../hive/ql/io/CombineHiveInputFormat.java | 4 +
      .../hadoop/hive/ql/io/HiveFileFormatUtils.java | 4 +-
      .../hadoop/hive/ql/io/HiveInputFormat.java | 7 +-
      .../ql/io/avro/AvroContainerOutputFormat.java | 2 +-
      .../hadoop/hive/ql/io/filters/BloomFilter.java | 298 -
      .../hive/ql/io/filters/BloomFilterIO.java | 44 +
      .../hadoop/hive/ql/io/filters/Murmur3.java | 334 -
      .../hadoop/hive/ql/io/merge/MergeFileTask.java | 6 +-
      .../hive/ql/io/orc/ColumnStatisticsImpl.java | 35 +-
      .../ql/io/orc/ConversionTreeReaderFactory.java | 38 +
      .../hive/ql/io/orc/DateColumnStatistics.java | 6 +-
      .../apache/hadoop/hive/ql/io/orc/FileDump.java | 99 +-
      .../hadoop/hive/ql/io/orc/IntegerWriter.java | 5 -
      .../hadoop/hive/ql/io/orc/JsonFileDump.java | 365 +
      .../hadoop/hive/ql/io/orc/MetadataReader.java | 1 -
      .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 4 +-
      .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 8 +-
      .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 24 +-
      .../apache/hadoop/hive/ql/io/orc/OrcUtils.java | 2 +-
      .../hive/ql/io/orc/RecordReaderFactory.java | 269 +
      .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 236 +-
      .../hive/ql/io/orc/RecordReaderUtils.java | 12 +-
      .../hive/ql/io/orc/RunLengthIntegerWriter.java | 9 +-
      .../ql/io/orc/RunLengthIntegerWriterV2.java | 11 +-
      .../hive/ql/io/orc/TreeReaderFactory.java | 98 +-
      .../hadoop/hive/ql/io/orc/WriterImpl.java | 15 +-
      .../ql/io/parquet/MapredParquetInputFormat.java | 7 +-
      .../io/parquet/MapredParquetOutputFormat.java | 7 +-
      .../parquet/VectorizedParquetInputFormat.java | 2 +-
      .../convert/HiveCollectionConverter.java | 23 +-
      .../io/parquet/convert/HiveGroupConverter.java | 10 -
      .../hive/ql/io/parquet/convert/Repeated.java | 26 +-
      .../read/ParquetRecordReaderWrapper.java | 7 +-
      .../serde/AbstractParquetMapInspector.java | 13 +-
      .../serde/DeepParquetHiveMapInspector.java | 6 +-
      .../serde/ParquetHiveArrayInspector.java | 43 +-
      .../serde/StandardParquetHiveMapInspector.java | 6 +-
      .../write/ParquetRecordWriterWrapper.java | 7 +-
      .../ql/io/rcfile/stats/PartialScanTask.java | 4 +-
      .../io/rcfile/truncate/ColumnTruncateTask.java | 4 +-
      .../hive/ql/io/sarg/SearchArgumentImpl.java | 81 +-
      .../hadoop/hive/ql/lib/PreOrderWalker.java | 8 +
      .../hadoop/hive/ql/lockmgr/DbLockManager.java | 41 +-
      .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 22 +-
      .../ql/lockmgr/zookeeper/ZooKeeperHiveLock.java | 22 +
      .../apache/hadoop/hive/ql/metadata/Hive.java | 30 +-
      .../apache/hadoop/hive/ql/metadata/Table.java | 18 +-
      .../formatting/JsonMetaDataFormatter.java | 2 +-
      .../formatting/MetaDataFormatUtils.java | 26 +
      .../metadata/formatting/MetaDataFormatter.java | 2 +-
      .../formatting/TextMetaDataFormatter.java | 76 +-
      .../BucketingSortingReduceSinkOptimizer.java | 13 +
      .../ql/optimizer/ColumnPrunerProcFactory.java | 25 +-
      .../optimizer/ConstantPropagateProcFactory.java | 226 +-
      .../hive/ql/optimizer/ConvertJoinMapJoin.java | 27 +-
      .../hive/ql/optimizer/GenMRTableScan1.java | 2 +-
      .../hive/ql/optimizer/GenMapRedUtils.java | 5 +-
      .../hive/ql/optimizer/GroupByOptimizer.java | 7 +-
      .../ql/optimizer/IdentityProjectRemover.java | 12 +
      .../ql/optimizer/LimitPushdownOptimizer.java | 9 +-
      .../ql/optimizer/NonBlockingOpDeDupProc.java | 13 +-
      .../hadoop/hive/ql/optimizer/Optimizer.java | 16 +-
      .../PrunerExpressionOperatorFactory.java | 3 -
      .../ql/optimizer/ReduceSinkMapJoinProc.java | 2 +-
      .../optimizer/RemoveDynamicPruningBySize.java | 19 +-
      .../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 -
      .../ql/optimizer/calcite/HiveCalciteUtil.java | 412 +-
      .../ql/optimizer/calcite/HiveConfigContext.java | 37 +
      .../calcite/HiveDefaultRelMetadataProvider.java | 55 +-
      .../ql/optimizer/calcite/HiveRelCollation.java | 34 +
      .../optimizer/calcite/HiveRelDistribution.java | 80 +
      .../ql/optimizer/calcite/RelOptHiveTable.java | 221 +-
      .../calcite/cost/HiveAlgorithmsConf.java | 39 +
      .../calcite/cost/HiveAlgorithmsUtil.java | 363 +
      .../ql/optimizer/calcite/cost/HiveCost.java | 52 +-
      .../optimizer/calcite/cost/HiveCostModel.java | 102 +
      .../ql/optimizer/calcite/cost/HiveCostUtil.java | 43 -
      .../calcite/cost/HiveDefaultCostModel.java | 123 +
      .../calcite/cost/HiveOnTezCostModel.java | 635 +
      .../optimizer/calcite/cost/HiveRelMdCost.java | 71 +
      .../calcite/cost/HiveVolcanoPlanner.java | 9 +-
      .../calcite/reloperators/HiveAggregate.java | 68 +-
      .../calcite/reloperators/HiveFilter.java | 4 +-
      .../calcite/reloperators/HiveGroupingID.java | 17 +
      .../calcite/reloperators/HiveJoin.java | 151 +-
      .../calcite/reloperators/HiveLimit.java | 4 +-
      .../calcite/reloperators/HiveProject.java | 7 +-
      .../calcite/reloperators/HiveSemiJoin.java | 79 +
      .../calcite/reloperators/HiveSortExchange.java | 84 +
      .../calcite/reloperators/HiveTableScan.java | 126 +-
      .../rules/HiveExpandDistinctAggregatesRule.java | 278 +
      .../rules/HiveInsertExchange4JoinRule.java | 134 +
      .../calcite/rules/HiveJoinAddNotNullRule.java | 197 +
      .../calcite/rules/HiveJoinToMultiJoinRule.java | 333 +
      .../calcite/rules/HivePreFilteringRule.java | 218 +
      .../calcite/rules/HiveRelFieldTrimmer.java | 75 +
      .../calcite/rules/HiveWindowingFixRule.java | 163 +
      .../calcite/stats/HiveRelMdCollation.java | 67 +
      .../calcite/stats/HiveRelMdDistribution.java | 56 +
      .../calcite/stats/HiveRelMdMemory.java | 102 +
      .../calcite/stats/HiveRelMdParallelism.java | 117 +
      .../calcite/stats/HiveRelMdRowCount.java | 1 -
      .../optimizer/calcite/stats/HiveRelMdSize.java | 148 +
      .../calcite/stats/HiveRelMdUniqueKeys.java | 1 -
      .../calcite/translator/ASTBuilder.java | 3 +-
      .../calcite/translator/ASTConverter.java | 24 +-
      .../calcite/translator/ExprNodeConverter.java | 245 +-
      .../calcite/translator/HiveGBOpConvUtil.java | 1226 ++
      .../calcite/translator/HiveOpConverter.java | 990 ++
      .../translator/HiveOpConverterPostProc.java | 175 +
      .../translator/PlanModifierForASTConv.java | 111 +-
      .../translator/PlanModifierForReturnPath.java | 41 +
      .../calcite/translator/PlanModifierUtil.java | 138 +
      .../calcite/translator/RexNodeConverter.java | 5 +-
      .../calcite/translator/TypeConverter.java | 4 +-
      .../correlation/ReduceSinkDeDuplication.java | 12 +-
      .../ql/optimizer/lineage/ExprProcFactory.java | 3 +-
      .../ql/optimizer/pcr/PcrExprProcFactory.java | 3 +-
      .../physical/GenSparkSkewJoinProcessor.java | 9 +-
      .../physical/SparkMapJoinResolver.java | 185 +-
      .../hive/ql/optimizer/physical/Vectorizer.java | 427 +-
      .../spark/SetSparkReducerParallelism.java | 30 +-
      .../spark/SparkReduceSinkMapJoinProc.java | 2 +-
      .../spark/SparkSkewJoinProcFactory.java | 57 +-
      .../optimizer/spark/SparkSkewJoinResolver.java | 5 +-
      .../stats/annotation/StatsRulesProcFactory.java | 289 +-
      .../hive/ql/parse/BaseSemanticAnalyzer.java | 96 +-
      .../hadoop/hive/ql/parse/CalcitePlanner.java | 365 +-
      .../ql/parse/ColumnStatsSemanticAnalyzer.java | 24 +-
      .../hive/ql/parse/DDLSemanticAnalyzer.java | 162 +-
      .../apache/hadoop/hive/ql/parse/EximUtil.java | 173 +-
      .../hive/ql/parse/ExportSemanticAnalyzer.java | 121 +-
      .../hadoop/hive/ql/parse/GenTezProcContext.java | 5 +-
      .../hadoop/hive/ql/parse/GenTezUtils.java | 21 +-
      .../apache/hadoop/hive/ql/parse/GenTezWork.java | 40 +-
      .../hadoop/hive/ql/parse/GenTezWorkWalker.java | 6 +
      .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 2 +
      .../apache/hadoop/hive/ql/parse/HiveParser.g | 58 +-
      .../hadoop/hive/ql/parse/IdentifiersParser.g | 8 +-
      .../hive/ql/parse/ImportSemanticAnalyzer.java | 768 +-
      .../hive/ql/parse/MetaDataExportListener.java | 2 +-
      .../hive/ql/parse/OptimizeTezProcContext.java | 14 +
      .../hadoop/hive/ql/parse/PTFTranslator.java | 14 +-
      .../apache/hadoop/hive/ql/parse/ParseUtils.java | 42 +-
      .../hive/ql/parse/ProcessAnalyzeTable.java | 4 +-
      .../org/apache/hadoop/hive/ql/parse/QB.java | 29 +-
      .../hadoop/hive/ql/parse/ReplicationSpec.java | 314 +
      .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 343 +-
      .../hive/ql/parse/TableAccessAnalyzer.java | 6 +-
      .../hadoop/hive/ql/parse/TaskCompiler.java | 4 +-
      .../hadoop/hive/ql/parse/TezCompiler.java | 19 +-
      .../hive/ql/parse/TypeCheckProcFactory.java | 6 +-
      .../hadoop/hive/ql/parse/UnparseTranslator.java | 2 +-
      .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 13 +-
      .../hadoop/hive/ql/parse/WindowingSpec.java | 9 +-
      .../hive/ql/parse/spark/GenSparkUtils.java | 35 +-
      .../parse/spark/SparkProcessAnalyzeTable.java | 2 +-
      .../hive/ql/plan/AbstractOperatorDesc.java | 2 +-
      .../hadoop/hive/ql/plan/AddPartitionDesc.java | 15 +
      .../apache/hadoop/hive/ql/plan/BaseWork.java | 1 +
      .../hadoop/hive/ql/plan/ColStatistics.java | 33 +-
      .../hadoop/hive/ql/plan/CreateTableDesc.java | 14 +
      .../hadoop/hive/ql/plan/DropTableDesc.java | 23 +-
      .../hive/ql/plan/ExprNodeConstantDesc.java | 15 +-
      .../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 39 +
      .../hadoop/hive/ql/plan/ExprNodeNullDesc.java | 69 -
      .../apache/hadoop/hive/ql/plan/FilterDesc.java | 4 +-
      .../apache/hadoop/hive/ql/plan/JoinDesc.java | 19 +
      .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 29 +-
      .../org/apache/hadoop/hive/ql/plan/MapWork.java | 12 +-
      .../hadoop/hive/ql/plan/PartitionDesc.java | 4 +-
      .../apache/hadoop/hive/ql/plan/PlanUtils.java | 78 +-
      .../hadoop/hive/ql/plan/ReduceSinkDesc.java | 27 +-
      .../apache/hadoop/hive/ql/plan/ReduceWork.java | 5 +-
      .../apache/hadoop/hive/ql/plan/SparkWork.java | 11 +-
      .../apache/hadoop/hive/ql/plan/Statistics.java | 15 +-
      .../apache/hadoop/hive/ql/plan/TableDesc.java | 5 +-
      .../hadoop/hive/ql/plan/TableScanDesc.java | 2 +-
      .../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 107 +
      .../hadoop/hive/ql/plan/ptf/BoundaryDef.java | 42 +-
      .../hadoop/hive/ql/plan/ptf/CurrentRowDef.java | 12 +-
      .../hive/ql/plan/ptf/RangeBoundaryDef.java | 38 +-
      .../hive/ql/plan/ptf/ValueBoundaryDef.java | 50 +-
      .../hadoop/hive/ql/plan/ptf/WindowFrameDef.java | 31 +-
      .../AuthorizationPreEventListener.java | 76 +-
      ...MultiPartitionAuthorizationProviderBase.java | 43 +
      .../MetaStoreAuthzAPIAuthorizerEmbedOnly.java | 3 +-
      .../StorageBasedAuthorizationProvider.java | 11 +-
      .../AuthorizationMetaStoreFilterHook.java | 9 +-
      .../authorization/plugin/HiveOperationType.java | 19 +-
      .../plugin/HivePrivilegeObjectUtils.java | 51 +
      .../plugin/sqlstd/Operation2Privilege.java | 17 +-
      .../SQLStdHiveAuthorizationValidator.java | 5 +
      .../hadoop/hive/ql/session/SessionState.java | 37 +-
      .../apache/hadoop/hive/ql/stats/StatsUtils.java | 261 +-
      .../hadoop/hive/ql/txn/compactor/Cleaner.java | 38 +-
      .../hive/ql/txn/compactor/CompactorMR.java | 9 +-
      .../hive/ql/txn/compactor/CompactorThread.java | 12 +-
      .../hadoop/hive/ql/txn/compactor/Initiator.java | 13 +-
      .../hadoop/hive/ql/txn/compactor/Worker.java | 45 +-
      .../org/apache/hadoop/hive/ql/udf/UDFCrc32.java | 75 +
      .../org/apache/hadoop/hive/ql/udf/UDFMd5.java | 88 +
      .../apache/hadoop/hive/ql/udf/UDFRegExp.java | 76 -
      .../org/apache/hadoop/hive/ql/udf/UDFSha1.java | 88 +
      .../hive/ql/udf/generic/GenericUDAFAverage.java | 43 +-
      .../ql/udf/generic/GenericUDAFCollectList.java | 15 +-
      .../ql/udf/generic/GenericUDAFCollectSet.java | 14 +-
      .../ql/udf/generic/GenericUDAFComputeStats.java | 74 +-
      .../ql/udf/generic/GenericUDAFFirstValue.java | 47 +-
      .../ql/udf/generic/GenericUDAFLastValue.java | 39 +-
      .../hive/ql/udf/generic/GenericUDAFMax.java | 51 +-
      .../hive/ql/udf/generic/GenericUDAFMin.java | 8 +-
      .../GenericUDAFMkCollectionEvaluator.java | 21 +-
      .../generic/GenericUDAFStreamingEvaluator.java | 85 +-
      .../hive/ql/udf/generic/GenericUDAFSum.java | 41 +-
      .../hadoop/hive/ql/udf/generic/GenericUDF.java | 38 +-
      .../ql/udf/generic/GenericUDFAddMonths.java | 5 +-
      .../hive/ql/udf/generic/GenericUDFArray.java | 5 +-
      .../hive/ql/udf/generic/GenericUDFBaseDTI.java | 18 +
      .../hive/ql/udf/generic/GenericUDFCbrt.java | 3 +-
      .../hive/ql/udf/generic/GenericUDFCoalesce.java | 2 +-
      .../ql/udf/generic/GenericUDFDateFormat.java | 125 +
      .../hive/ql/udf/generic/GenericUDFDecode.java | 26 +-
      .../hive/ql/udf/generic/GenericUDFEncode.java | 2 +-
      .../ql/udf/generic/GenericUDFFactorial.java | 3 +-
      .../hive/ql/udf/generic/GenericUDFGreatest.java | 1 +
      .../hive/ql/udf/generic/GenericUDFInstr.java | 2 +-
      .../hive/ql/udf/generic/GenericUDFLastDay.java | 4 +-
      .../ql/udf/generic/GenericUDFLevenshtein.java | 5 +-
      .../hive/ql/udf/generic/GenericUDFLocate.java | 2 +-
      .../hive/ql/udf/generic/GenericUDFMap.java | 20 +-
      .../hive/ql/udf/generic/GenericUDFNextDay.java | 11 +-
      .../ql/udf/generic/GenericUDFParamUtils.java | 71 +
      .../hive/ql/udf/generic/GenericUDFPrintf.java | 3 +-
      .../hive/ql/udf/generic/GenericUDFQuarter.java | 85 +
      .../hive/ql/udf/generic/GenericUDFRegExp.java | 133 +
      .../hive/ql/udf/generic/GenericUDFSha2.java | 137 +
      .../ql/udf/generic/GenericUDFSortArray.java | 17 +-
      .../hive/ql/udf/generic/GenericUDFSoundex.java | 6 +-
      .../udf/generic/GenericUDFSubstringIndex.java | 159 +
      .../ql/udf/generic/GenericUDFTimestamp.java | 24 +
      .../udf/generic/GenericUDFToUnixTimeStamp.java | 6 +-
      .../ql/udf/generic/GenericUDFTranslate.java | 8 +-
      .../hive/ql/udf/generic/GenericUDFTrunc.java | 11 +-
      .../ql/udf/generic/GenericUDFUnixTimeStamp.java | 36 +-
      .../hive/ql/udf/generic/GenericUDFUtils.java | 6 +-
      .../hadoop/hive/ql/udf/ptf/MatchPath.java | 5 +-
      .../hive/ql/udf/ptf/WindowingTableFunction.java | 39 +-
      .../apache/hadoop/hive/ql/TestTxnCommands2.java | 77 +-
      .../session/TestSparkSessionManagerImpl.java | 5 +
      .../exec/vector/mapjoin/TestDebugDisplay.java | 63 +
      .../mapjoin/fast/CommonFastHashTable.java | 128 +
      .../mapjoin/fast/RandomByteArrayStream.java | 92 +
      .../vector/mapjoin/fast/RandomLongStream.java | 49 +
      .../fast/TestVectorMapJoinFastLongHashMap.java | 219 +
      .../TestVectorMapJoinFastMultiKeyHashMap.java | 231 +
      .../TestVectorMapJoinRowBytesContainer.java | 74 +
      .../hive/ql/io/filters/TestBloomFilter.java | 458 -
      .../hadoop/hive/ql/io/filters/TestMurmur3.java | 189 -
      .../hive/ql/io/orc/TestColumnStatistics.java | 20 +-
      .../hadoop/hive/ql/io/orc/TestFileDump.java | 2 +-
      .../hadoop/hive/ql/io/orc/TestJsonFileDump.java | 138 +
      .../hadoop/hive/ql/io/orc/TestOrcFile.java | 105 +-
      .../hive/ql/io/orc/TestOrcSerDeStats.java | 12 +-
      .../hadoop/hive/ql/io/orc/TestOrcTimezone1.java | 4 +-
      .../hadoop/hive/ql/io/orc/TestOrcTimezone2.java | 2 +-
      .../hive/ql/io/orc/TestRecordReaderImpl.java | 328 +-
      .../io/parquet/AbstractTestParquetDirect.java | 23 +-
      .../ql/io/parquet/TestArrayCompatibility.java | 62 +-
      .../ql/io/parquet/TestDataWritableWriter.java | 44 +-
      .../hive/ql/io/parquet/TestMapStructures.java | 54 +-
      .../hive/ql/io/parquet/TestParquetSerDe.java | 8 +-
      .../serde/TestAbstractParquetMapInspector.java | 4 +-
      .../serde/TestDeepParquetHiveMapInspector.java | 4 +-
      .../serde/TestParquetHiveArrayInspector.java | 3 +-
      .../TestStandardParquetHiveMapInspector.java | 4 +-
      .../hive/ql/io/sarg/TestSearchArgumentImpl.java | 131 +-
      .../hive/ql/lockmgr/TestDbTxnManager.java | 2 +
      .../hive/ql/lockmgr/TestDbTxnManager2.java | 233 +
      .../ql/optimizer/physical/TestVectorizer.java | 2 +-
      .../apache/hadoop/hive/ql/parse/TestIUD.java | 7 +
      ...tedCharsInColumnNameCreateTableNegative.java | 87 +
      .../hadoop/hive/ql/plan/TestViewEntity.java | 141 +
      .../hadoop/hive/ql/session/TestAddResource.java | 116 +-
      .../hive/ql/session/TestSessionState.java | 30 +-
      .../hive/ql/txn/compactor/TestCleaner.java | 56 +-
      .../hive/ql/txn/compactor/TestInitiator.java | 63 +-
      .../hive/ql/txn/compactor/TestWorker.java | 45 +
      .../hadoop/hive/ql/udaf/TestStreamingSum.java | 16 +-
      .../hadoop/hive/ql/udf/TestGenericUDFDate.java | 92 -
      .../hive/ql/udf/TestGenericUDFDateAdd.java | 145 -
      .../hive/ql/udf/TestGenericUDFDateDiff.java | 116 -
      .../hive/ql/udf/TestGenericUDFDateSub.java | 143 -
      .../hadoop/hive/ql/udf/TestGenericUDFUtils.java | 58 -
      .../apache/hadoop/hive/ql/udf/TestUDFCrc32.java | 74 +
      .../apache/hadoop/hive/ql/udf/TestUDFJson.java | 18 +
      .../apache/hadoop/hive/ql/udf/TestUDFMd5.java | 57 +
      .../apache/hadoop/hive/ql/udf/TestUDFSha1.java | 57 +
      .../hive/ql/udf/generic/TestGenericUDFDate.java | 92 +
      .../ql/udf/generic/TestGenericUDFDateAdd.java | 143 +
      .../ql/udf/generic/TestGenericUDFDateDiff.java | 116 +
      .../udf/generic/TestGenericUDFDateFormat.java | 173 +
      .../ql/udf/generic/TestGenericUDFDateSub.java | 143 +
      .../udf/generic/TestGenericUDFLevenshtein.java | 4 +-
      .../ql/udf/generic/TestGenericUDFNextDay.java | 4 +-
      .../ql/udf/generic/TestGenericUDFQuarter.java | 182 +
      .../ql/udf/generic/TestGenericUDFRegexp.java | 135 +
      .../hive/ql/udf/generic/TestGenericUDFSha2.java | 271 +
      .../ql/udf/generic/TestGenericUDFSortArray.java | 158 +
      .../generic/TestGenericUDFSubstringIndex.java | 97 +
      .../ql/udf/generic/TestGenericUDFUtils.java | 57 +
      .../authorization_set_nonexistent_conf.q | 7 +
      .../queries/clientnegative/insertsel_fail.q | 1 +
      .../clientnegative/protectmode_part_no_drop2.q | 11 +
      .../table_nonprintable_negative.q | 11 +
      .../udaf_collect_set_unsupported.q | 3 +
      .../clientnegative/udf_sort_array_wrong3.q | 2 +-
      .../clientpositive/alter_change_db_location.q | 5 +
      .../queries/clientpositive/alter_merge_2_orc.q | 6 +-
      .../clientpositive/alter_partition_coltype.q | 1 +
      .../clientpositive/alter_table_partition_drop.q | 11 +
      .../clientpositive/annotate_stats_part.q | 1 +
      .../clientpositive/authorization_view_sqlstd.q | 4 +
      ql/src/test/queries/clientpositive/auto_join0.q | 11 +-
      ql/src/test/queries/clientpositive/auto_join1.q | 2 +
      .../clientpositive/auto_sortmerge_join_13.q | 2 +
      .../clientpositive/auto_sortmerge_join_5.q | 3 +
      .../queries/clientpositive/autogen_colalias.q | 4 +-
      .../test/queries/clientpositive/bucket_many.q | 16 +
      ql/src/test/queries/clientpositive/cbo_gby.q | 3 +
      ql/src/test/queries/clientpositive/cbo_join.q | 1 +
      ql/src/test/queries/clientpositive/cbo_limit.q | 2 +-
      .../queries/clientpositive/cbo_rp_auto_join0.q | 24 +
      .../queries/clientpositive/cbo_rp_auto_join1.q | 274 +
      .../queries/clientpositive/cbo_simple_select.q | 3 +-
      .../test/queries/clientpositive/cbo_subq_in.q | 8 +-
      .../queries/clientpositive/cbo_subq_not_in.q | 4 +-
      .../test/queries/clientpositive/cbo_udf_udaf.q | 14 +-
      .../test/queries/clientpositive/cbo_windowing.q | 2 +
      .../queries/clientpositive/compute_stats_date.q | 28 +
      .../clientpositive/cross_product_check_1.q | 9 +-
      .../clientpositive/cross_product_check_2.q | 6 +-
      .../clientpositive/crtseltbl_serdeprops.q | 11 +
      ql/src/test/queries/clientpositive/ctas.q | 1 +
      ql/src/test/queries/clientpositive/cteViews.q | 41 +
      .../queries/clientpositive/decimal_precision2.q | 15 +
      .../dynamic_partition_pruning_2.q | 17 +
      .../clientpositive/dynpart_sort_optimization2.q | 7 +-
      .../clientpositive/encryption_drop_table.q | 18 +
      .../clientpositive/encryption_drop_view.q | 6 +
      .../encryption_insert_partition_dynamic.q | 8 +-
      .../encryption_insert_partition_static.q | 41 +-
      .../clientpositive/encryption_insert_values.q | 15 +
      .../encryption_join_unencrypted_tbl.q | 4 +-
      ...yption_join_with_different_encryption_keys.q | 8 +-
      .../encryption_load_data_to_encrypted_tables.q | 4 +-
      .../clientpositive/encryption_move_tbl.q | 20 +
      .../encryption_select_read_only_encrypted_tbl.q | 4 +-
      .../clientpositive/exec_parallel_column_stats.q | 5 +
      .../test/queries/clientpositive/explainuser_2.q | 30 +-
      .../extrapolate_part_stats_partial.q | 2 +
      .../extrapolate_part_stats_partial_ndv.q | 2 +
      .../clientpositive/filter_cond_pushdown.q | 19 +
      .../clientpositive/filter_join_breaktask2.q | 2 +
      ql/src/test/queries/clientpositive/fold_case.q | 20 +
      .../clientpositive/fold_eq_with_case_when.q | 21 +
      ql/src/test/queries/clientpositive/fold_when.q | 31 +
      .../clientpositive/hybridgrace_hashjoin_1.q | 258 +
      .../clientpositive/hybridgrace_hashjoin_2.q | 152 +
      .../queries/clientpositive/hybridhashjoin.q | 250 -
      .../test/queries/clientpositive/insert_into1.q | 15 +-
      .../test/queries/clientpositive/insert_into2.q | 14 +-
      .../clientpositive/insert_into_with_schema2.q | 34 +
      .../clientpositive/insert_nonacid_from_acid.q | 2 +
      .../clientpositive/insert_overwrite_directory.q | 141 +
      .../join_merge_multi_expressions.q | 1 +
      .../test/queries/clientpositive/join_nullsafe.q | 2 +
      .../queries/clientpositive/join_on_varchar.q | 12 +
      .../clientpositive/json_serde_qualified_types.q | 12 +
      .../queries/clientpositive/limit_pushdown.q | 4 +
      .../queries/clientpositive/mapjoin_decimal.q | 6 +-
      .../queries/clientpositive/mapjoin_mapjoin.q | 1 +
      ql/src/test/queries/clientpositive/mergejoin.q | 124 +
      ql/src/test/queries/clientpositive/mrr.q | 2 +
      .../queries/clientpositive/optimize_nullscan.q | 4 +
      .../clientpositive/orc_int_type_promotion.q | 79 +
      ql/src/test/queries/clientpositive/orc_merge1.q | 2 +
      .../clientpositive/partition_coltype_literals.q | 75 +
      .../queries/clientpositive/partition_date2.q | 2 +
      .../test/queries/clientpositive/repl_1_drop.q | 84 +
      .../queries/clientpositive/repl_2_exim_basic.q | 79 +
      .../clientpositive/repl_3_exim_metadata.q | 40 +
      .../runtime_skewjoin_mapjoin_spark.q | 24 +
      .../queries/clientpositive/script_env_var1.q | 4 +-
      .../queries/clientpositive/selectDistinctStar.q | 2 +
      ql/src/test/queries/clientpositive/skewjoin.q | 11 +-
      .../queries/clientpositive/stats_only_null.q | 3 +
      .../queries/clientpositive/subquery_exists.q | 1 +
      .../queries/clientpositive/table_nonprintable.q | 30 +
      ql/src/test/queries/clientpositive/temp_table.q | 6 +-
      ql/src/test/queries/clientpositive/tez_dml.q | 2 +
      .../test/queries/clientpositive/tez_join_hash.q | 4 +
      .../queries/clientpositive/tez_joins_explain.q | 1 -
      .../queries/clientpositive/tez_multi_union.q | 1 +
      .../clientpositive/tez_schema_evolution.q | 12 +-
      ql/src/test/queries/clientpositive/tez_smb_1.q | 1 +
      .../test/queries/clientpositive/tez_smb_main.q | 7 +
      ql/src/test/queries/clientpositive/tez_union.q | 18 +
      .../clientpositive/tez_union_multiinsert.q | 120 +
      .../clientpositive/timestamp_ints_casts.q | 74 +
      .../queries/clientpositive/udaf_collect_set_2.q | 222 +
      .../clientpositive/udaf_percentile_approx_23.q | 2 +-
      ql/src/test/queries/clientpositive/udf_crc32.q | 13 +
      .../queries/clientpositive/udf_date_format.q | 60 +
      .../clientpositive/udf_folder_constants.q | 11 +
      ql/src/test/queries/clientpositive/udf_md5.q | 13 +
      .../test/queries/clientpositive/udf_quarter.q | 100 +
      ql/src/test/queries/clientpositive/udf_sha1.q | 13 +
      ql/src/test/queries/clientpositive/udf_sha2.q | 41 +
      .../queries/clientpositive/udf_sort_array.q | 10 +
      .../clientpositive/udf_substring_index.q | 32 +
      .../queries/clientpositive/udf_unix_timestamp.q | 8 +-
      ql/src/test/queries/clientpositive/union12.q | 2 +
      ql/src/test/queries/clientpositive/union17.q | 2 +-
      ql/src/test/queries/clientpositive/union20.q | 2 +-
      ql/src/test/queries/clientpositive/union21.q | 2 +-
      ql/src/test/queries/clientpositive/union27.q | 2 +-
      .../queries/clientpositive/unionDistinct_2.q | 2 +
      .../queries/clientpositive/union_remove_22.q | 2 +
      .../clientpositive/union_remove_6_subq.q | 2 +
      .../update_after_multiple_inserts.q | 2 +
      .../clientpositive/update_all_partitioned.q | 4 +-
      .../clientpositive/update_where_partitioned.q | 4 +-
      .../queries/clientpositive/vector_aggregate_9.q | 2 +
      .../clientpositive/vector_binary_join_groupby.q | 55 +
      .../clientpositive/vector_char_mapjoin1.q | 9 +
      .../queries/clientpositive/vector_data_types.q | 12 +-
      .../clientpositive/vector_decimal_mapjoin.q | 43 +-
      .../clientpositive/vector_grouping_sets.q | 63 +
      .../queries/clientpositive/vector_inner_join.q | 61 +
      .../clientpositive/vector_interval_mapjoin.q | 87 +
      .../test/queries/clientpositive/vector_join30.q | 160 +
      .../clientpositive/vector_join_filters.q | 38 +
      .../queries/clientpositive/vector_join_nulls.q | 33 +
      .../clientpositive/vector_left_outer_join.q | 1 +
      .../clientpositive/vector_left_outer_join2.q | 64 +
      .../clientpositive/vector_leftsemi_mapjoin.q | 403 +
      .../vector_mr_diff_schema_alias.q | 115 +
      .../clientpositive/vector_null_projection.q | 20 +
      .../clientpositive/vector_nullsafe_join.q | 72 +
      .../queries/clientpositive/vector_outer_join0.q | 25 +
      .../queries/clientpositive/vector_outer_join1.q | 64 +
      .../queries/clientpositive/vector_outer_join2.q | 39 +
      .../queries/clientpositive/vector_outer_join3.q | 80 +
      .../queries/clientpositive/vector_outer_join4.q | 66 +
      .../queries/clientpositive/vector_outer_join5.q | 173 +
      .../clientpositive/vectorization_part_project.q | 4 +-
      .../queries/clientpositive/vectorized_casts.q | 2 +-
      .../queries/clientpositive/vectorized_context.q | 1 +
      .../clientpositive/vectorized_distinct_gby.q | 2 +-
      .../queries/clientpositive/vectorized_mapjoin.q | 2 +
      .../clientpositive/vectorized_nested_mapjoin.q | 2 +
      .../clientpositive/vectorized_rcfile_columnar.q | 2 +-
      .../clientpositive/vectorized_shufflejoin.q | 10 +-
      .../clientpositive/vectorized_timestamp_funcs.q | 41 +-
      .../vectorized_timestamp_ints_casts.q | 76 +
      .../queries/clientpositive/windowing_navfn.q | 4 +-
      .../clientpositive/windowing_windowspec2.q | 60 +
      ql/src/test/resources/RefreshedJarClassV1.txt | 26 +
      ql/src/test/resources/RefreshedJarClassV2.txt | 26 +
      ql/src/test/resources/SessionStateTest.jar.v1 | Bin 2176 -> 0 bytes
      ql/src/test/resources/SessionStateTest.jar.v2 | Bin 2177 -> 0 bytes
      ql/src/test/resources/orc-file-dump.json | 1354 ++
      ql/src/test/resources/orc-file-has-null.out | 42 +-
      .../alter_rename_partition_failure3.q.out | 2 +-
      .../alter_table_add_partition.q.out | 2 +-
      .../clientnegative/alter_view_failure5.q.out | 2 +-
      .../clientnegative/alter_view_failure7.q.out | 2 +-
      .../clientnegative/archive_partspec1.q.out | 2 +-
      .../clientnegative/archive_partspec5.q.out | 8 +-
      .../clientnegative/authorization_ctas2.q.out | 2 +
      .../authorization_set_nonexistent_conf.q.out | 9 +
      .../results/clientnegative/insertsel_fail.q.out | 1 +
      .../clientnegative/join_nonexistent_part.q.out | 2 +-
      .../protectmode_part_no_drop2.q.out | 51 +
      .../table_nonprintable_negative.q.out | 19 +
      ql/src/test/results/clientnegative/touch2.q.out | 2 +-
      .../truncate_partition_column.q.out | 2 +-
      .../udaf_collect_set_unsupported.q.out | 1 +
      .../clientnegative/udf_add_months_error_1.q.out | 2 +-
      .../clientnegative/udf_last_day_error_1.q.out | 2 +-
      .../clientnegative/udf_next_day_error_1.q.out | 2 +-
      .../clientnegative/udf_next_day_error_2.q.out | 2 +-
      .../clientnegative/udf_sort_array_wrong1.q.out | 2 +-
      .../clientnegative/udf_sort_array_wrong2.q.out | 2 +-
      .../clientnegative/udf_sort_array_wrong3.q.out | 2 +-
      .../results/clientpositive/add_part_exist.q.out | 2 +
      ql/src/test/results/clientpositive/alter1.q.out | 4 +
      ql/src/test/results/clientpositive/alter2.q.out | 4 +
      ql/src/test/results/clientpositive/alter3.q.out | 4 +
      ql/src/test/results/clientpositive/alter4.q.out | 4 +
      ql/src/test/results/clientpositive/alter5.q.out | 4 +
      .../alter_change_db_location.q.out | 37 +
      .../results/clientpositive/alter_db_owner.q.out | 6 +
      .../results/clientpositive/alter_index.q.out | 2 +
      .../clientpositive/alter_merge_2_orc.q.out | 16 +-
      .../clientpositive/alter_rename_partition.q.out | 6 +
      .../alter_table_partition_drop.q.out | 74 +
      .../clientpositive/annotate_stats_filter.q.out | 28 +-
      .../clientpositive/annotate_stats_join.q.out | 40 +-
      .../annotate_stats_join_pkfk.q.out | 42 +-
      .../clientpositive/annotate_stats_limit.q.out | 2 +-
      .../clientpositive/annotate_stats_part.q.out | 10 +-
      .../clientpositive/annotate_stats_select.q.out | 8 +-
      .../clientpositive/annotate_stats_table.q.out | 4 +-
      .../authorization_owner_actions_db.q.out | 2 +
      .../authorization_view_sqlstd.q.out | 14 +
      .../results/clientpositive/auto_join0.q.out | 235 +-
      .../results/clientpositive/auto_join1.q.out | 8 +-
      .../results/clientpositive/auto_join16.q.out | 4 +-
      .../results/clientpositive/auto_join30.q.out | 24 +
      .../results/clientpositive/auto_join31.q.out | 4 +
      .../results/clientpositive/auto_join32.q.out | 56 +-
      .../results/clientpositive/auto_join8.q.out | 14 +-
      .../clientpositive/auto_join_stats.q.out | 28 +-
      .../clientpositive/auto_join_stats2.q.out | 8 +-
      .../auto_join_without_localtask.q.out | 60 +
      .../clientpositive/auto_sortmerge_join_1.q.out | 11 +
      .../clientpositive/auto_sortmerge_join_12.q.out | 2 +-
      .../clientpositive/auto_sortmerge_join_14.q.out | 4 +
      .../clientpositive/auto_sortmerge_join_15.q.out | 4 +
      .../clientpositive/auto_sortmerge_join_2.q.out | 10 +
      .../clientpositive/auto_sortmerge_join_3.q.out | 11 +
      .../clientpositive/auto_sortmerge_join_4.q.out | 11 +
      .../clientpositive/auto_sortmerge_join_5.q.out | 17 +
      .../clientpositive/auto_sortmerge_join_6.q.out | 66 +
      .../clientpositive/auto_sortmerge_join_7.q.out | 11 +
      .../clientpositive/auto_sortmerge_join_8.q.out | 11 +
      .../clientpositive/auto_sortmerge_join_9.q.out | 143 +
      .../clientpositive/autogen_colalias.q.out | 12 +-
      .../clientpositive/binarysortable_1.q.out | Bin 4302 -> 4329 bytes
      .../results/clientpositive/bucket_many.q.out | 230 +
      .../clientpositive/bucket_map_join_1.q.out | 10 +-
      .../clientpositive/bucket_map_join_2.q.out | 10 +-
      .../clientpositive/bucketcontext_1.q.out | 1 +
      .../clientpositive/bucketcontext_2.q.out | 1 +
      .../clientpositive/bucketcontext_3.q.out | 1 +
      .../clientpositive/bucketcontext_4.q.out | 1 +
      .../clientpositive/bucketcontext_5.q.out | 1 +
      .../clientpositive/bucketcontext_6.q.out | 1 +
      .../clientpositive/bucketcontext_7.q.out | 1 +
      .../clientpositive/bucketcontext_8.q.out | 1 +
      .../results/clientpositive/bucketmapjoin1.q.out | 96 +-
      .../bucketsortoptimize_insert_2.q.out | 20 +
      .../bucketsortoptimize_insert_4.q.out | 16 +
      .../bucketsortoptimize_insert_5.q.out | 20 +
      .../bucketsortoptimize_insert_6.q.out | 24 +
      .../test/results/clientpositive/cbo_gby.q.out | 18 +-
      .../test/results/clientpositive/cbo_join.q.out | 350 +-
      .../test/results/clientpositive/cbo_limit.q.out | 4 +-
      .../clientpositive/cbo_rp_auto_join0.q.out | 258 +
      .../clientpositive/cbo_rp_auto_join1.q.out | 1512 ++
      .../clientpositive/cbo_simple_select.q.out | 10 +
      .../results/clientpositive/cbo_subq_in.q.out | 38 +-
      .../clientpositive/cbo_subq_not_in.q.out | 32 +-
      .../results/clientpositive/cbo_udf_udaf.q.out | 46 +-
      .../results/clientpositive/cbo_windowing.q.out | 66 +-
      .../clientpositive/column_access_stats.q.out | 78 +-
      .../clientpositive/columnstats_partlvl.q.out | 14 +-
      .../clientpositive/columnstats_partlvl_dp.q.out | 8 +-
      .../clientpositive/columnstats_tbllvl.q.out | 14 +-
      .../results/clientpositive/complex_alias.q.out | 42 +-
      .../clientpositive/compute_stats_date.q.out | 127 +
      .../clientpositive/correlationoptimizer1.q.out | 16 +-
      .../clientpositive/correlationoptimizer10.q.out | 6 +-
      .../clientpositive/correlationoptimizer11.q.out | 8 +-
      .../clientpositive/correlationoptimizer12.q.out | 8 +-
      .../clientpositive/correlationoptimizer15.q.out | 6 +-
      .../clientpositive/correlationoptimizer2.q.out | 10 +-
      .../clientpositive/correlationoptimizer3.q.out | 72 +-
      .../clientpositive/correlationoptimizer4.q.out | 6 +-
      .../clientpositive/correlationoptimizer5.q.out | 6 +
      .../clientpositive/correlationoptimizer6.q.out | 80 +-
      .../results/clientpositive/create_view.q.out | 2 +
      .../clientpositive/cross_product_check_1.q.out | 20 +-
      .../clientpositive/cross_product_check_2.q.out | 76 +-
      .../clientpositive/crtseltbl_serdeprops.q.out | 35 +
      ql/src/test/results/clientpositive/ctas.q.out | 2 +
      .../results/clientpositive/ctas_colname.q.out | 8 +-
      .../test/results/clientpositive/cteViews.q.out | 242 +
      .../test/results/clientpositive/database.q.out | 12 +
      .../clientpositive/database_location.q.out | 8 +
      .../clientpositive/database_properties.q.out | 6 +
      .../clientpositive/dbtxnmgr_query5.q.out | 2 +
      .../clientpositive/decimal_precision2.q.out | 163 +
      .../results/clientpositive/decimal_udf.q.out | 2 +-
      .../clientpositive/describe_database.q.out | 4 +
      .../clientpositive/describe_database_json.q.out | 12 +
      .../clientpositive/describe_table_json.q.out | 4 +
      .../display_colstats_tbllvl.q.out | 6 +-
      .../dynpart_sort_optimization2.q.out | 166 +-
      .../encrypted/encryption_drop_table.q.out | 62 +
      .../encrypted/encryption_drop_view.q.out | 34 +
      .../encryption_insert_partition_dynamic.q.out | 32 +-
      .../encryption_insert_partition_static.q.out | 385 +-
      .../encrypted/encryption_insert_values.q.out | 71 +
      .../encryption_join_unencrypted_tbl.q.out | 16 +-
      ...on_join_with_different_encryption_keys.q.out | 16 +-
      ...cryption_load_data_to_encrypted_tables.q.out | 8 +-
      .../encrypted/encryption_move_tbl.q.out | 55 +
      ...ryption_select_read_only_encrypted_tbl.q.out | 8 +-
      .../exec_parallel_column_stats.q.out | 51 +
      .../clientpositive/explain_logical.q.out | 28 +-
      .../clientpositive/explain_rearrange.q.out | 32 +-
      .../clientpositive/filter_cond_pushdown.q.out | 382 +
      .../clientpositive/filter_join_breaktask2.q.out | 8 +-
      .../results/clientpositive/filter_numeric.q.out | 6 +-
      .../test/results/clientpositive/fold_case.q.out | 477 +
      .../clientpositive/fold_eq_with_case_when.q.out | 231 +
      .../test/results/clientpositive/fold_when.q.out | 480 +
      .../results/clientpositive/groupby_cube1.q.out | 108 +-
      .../clientpositive/groupby_grouping_sets2.q.out | 36 +-
      .../clientpositive/groupby_grouping_sets3.q.out | 32 +-
      .../clientpositive/groupby_grouping_sets4.q.out | 72 +-
      .../clientpositive/groupby_grouping_sets5.q.out | 44 +-
      .../clientpositive/groupby_grouping_sets6.q.out | 30 +-
      .../groupby_grouping_window.q.out | 4 +-
      .../results/clientpositive/groupby_ppd.q.out | 34 +-
      .../clientpositive/groupby_resolution.q.out | 4 +-
      .../clientpositive/groupby_rollup1.q.out | 94 +-
      .../results/clientpositive/groupby_sort_6.q.out | 80 +-
      .../test/results/clientpositive/having2.q.out | 32 +-
      .../identity_project_remove_skip.q.out | 6 +
      .../results/clientpositive/implicit_cast1.q.out | 8 +-
      .../clientpositive/index_auto_empty.q.out | 8 +-
      .../results/clientpositive/index_creation.q.out | 2 +
      .../results/clientpositive/index_serde.q.out | 6 +-
      .../clientpositive/index_skewtable.q.out | 12 +-
      .../clientpositive/infer_const_type.q.out | 28 +-
      ql/src/test/results/clientpositive/input2.q.out | 4 +
      .../test/results/clientpositive/input21.q.out | 10 +-
      .../test/results/clientpositive/input23.q.out | 22 +-
      .../test/results/clientpositive/input24.q.out | 4 +-
      .../test/results/clientpositive/input25.q.out | 36 +-
      .../test/results/clientpositive/input26.q.out | 36 +-
      ql/src/test/results/clientpositive/input3.q.out | 4 +
      ql/src/test/results/clientpositive/input6.q.out | 2 +-
      ql/src/test/results/clientpositive/input8.q.out | 2 +-
      ql/src/test/results/clientpositive/input9.q.out | 8 +-
      .../results/clientpositive/input_part4.q.out | 6 +-
      .../test/results/clientpositive/insert1.q.out | 46 +-
      .../results/clientpositive/insert_into1.q.out | 86 +-
      .../results/clientpositive/insert_into2.q.out | 82 +-
      .../insert_into_with_schema2.q.out | 135 +
      .../insert_nonacid_from_acid.q.out | 20 +-
      .../insert_overwrite_directory.q.out | 1813 +++
      ql/src/test/results/clientpositive/join19.q.out | 62 +-
      ql/src/test/results/clientpositive/join29.q.out | 6 +
      ql/src/test/results/clientpositive/join31.q.out | 6 +
      ql/src/test/results/clientpositive/join8.q.out | 16 +-
      .../join_cond_pushdown_unqual1.q.out | 48 +-
      .../join_cond_pushdown_unqual2.q.out | 48 +-
      .../join_cond_pushdown_unqual3.q.out | 48 +-
      .../join_cond_pushdown_unqual4.q.out | 48 +-
      .../results/clientpositive/join_hive_626.q.out | 28 +-
      .../join_merge_multi_expressions.q.out | 6 +-
      .../results/clientpositive/join_nullsafe.q.out | 16 +-
      .../clientpositive/join_on_varchar.q.out | 146 +
      .../results/clientpositive/join_reorder.q.out | 92 +-
      .../results/clientpositive/join_reorder2.q.out | 68 +-
      .../results/clientpositive/join_reorder3.q.out | 68 +-
      .../results/clientpositive/join_reorder4.q.out | 54 +-
      .../test/results/clientpositive/join_view.q.out | 31 +-
      .../json_serde_qualified_types.q.out | 33 +
      .../test/results/clientpositive/keyword_1.q.out | 8 +-
      .../clientpositive/lateral_view_explode2.q.out | 6 +-
      .../results/clientpositive/limit_pushdown.q.out | 88 +
      .../list_bucket_query_oneskew_2.q.out | 22 +-
      .../clientpositive/literal_decimal.q.out | 6 +-
      .../clientpositive/load_dyn_part14.q.out | 24 +-
      .../clientpositive/mapjoin_subquery2.q.out | 20 +-
      .../results/clientpositive/merge_join_1.q.out | 32 +-
      .../test/results/clientpositive/mergejoin.q.out | 3157 ++++
      .../results/clientpositive/mergejoins.q.out | 40 +-
      .../clientpositive/mergejoins_mixed.q.out | 332 +-
      .../results/clientpositive/metadataonly1.q.out | 88 +-
      .../results/clientpositive/multiMapJoin1.q.out | 76 +
      .../results/clientpositive/multiMapJoin2.q.out | 30 +-
      .../clientpositive/multigroupby_singlemr.q.out | 136 +-
      .../results/clientpositive/nullformatCTAS.q.out | 6 +-
      .../results/clientpositive/nullgroup3.q.out | 16 +-
      .../results/clientpositive/nullgroup5.q.out | 23 +-
      .../clientpositive/num_op_type_conv.q.out | 6 +-
      .../clientpositive/optimize_nullscan.q.out | 331 +-
      .../clientpositive/orc_int_type_promotion.q.out | 377 +
      .../results/clientpositive/orc_merge1.q.out | 8 +-
      .../clientpositive/orc_predicate_pushdown.q.out | 36 +-
      .../clientpositive/partition_boolexpr.q.out | 32 +
      .../partition_coltype_literals.q.out | 647 +
      .../clientpositive/partition_date2.q.out | 8 +-
      .../clientpositive/partition_timestamp.q.out | 116 +-
      .../clientpositive/partition_timestamp2.q.out | 250 +-
      .../clientpositive/ppd_constant_expr.q.out | 4 +-
      .../test/results/clientpositive/ppd_gby.q.out | 41 +-
      .../test/results/clientpositive/ppd_gby2.q.out | 49 +-
      .../results/clientpositive/ppd_gby_join.q.out | 68 +-
      .../test/results/clientpositive/ppd_join.q.out | 64 +-
      .../test/results/clientpositive/ppd_join2.q.out | 81 +-
      .../test/results/clientpositive/ppd_join3.q.out | 91 +-
      .../test/results/clientpositive/ppd_join4.q.out | 24 +-
      .../test/results/clientpositive/ppd_join5.q.out | 22 +-
      .../clientpositive/ppd_outer_join2.q.out | 16 +-
      .../clientpositive/ppd_outer_join3.q.out | 16 +-
      .../clientpositive/ppd_outer_join4.q.out | 32 +-
      .../clientpositive/ppd_outer_join5.q.out | 72 +-
      .../clientpositive/ppd_repeated_alias.q.out | 78 +-
      .../results/clientpositive/ppd_udf_col.q.out | 190 +-
      .../test/results/clientpositive/ppd_union.q.out | 42 +-
      .../results/clientpositive/ppd_union_view.q.out | 98 +-
      ql/src/test/results/clientpositive/ppd_vc.q.out | 8 +-
      ql/src/test/results/clientpositive/ptf.q.out | 160 +-
      .../results/clientpositive/ptf_streaming.q.out | 84 +-
      .../clientpositive/ql_rewrite_gbtoidx.q.out | 258 +-
      .../ql_rewrite_gbtoidx_cbo_1.q.out | 260 +-
      .../ql_rewrite_gbtoidx_cbo_2.q.out | 18 +-
      .../query_result_fileformat.q.out | 12 +-
      ql/src/test/results/clientpositive/quote1.q.out | 6 +-
      .../results/clientpositive/quotedid_basic.q.out | 78 +-
      .../results/clientpositive/quotedid_skew.q.out | 40 +-
      .../clientpositive/reduce_deduplicate.q.out | 40 +-
      .../results/clientpositive/rename_column.q.out | 4 +
      .../results/clientpositive/repl_1_drop.q.out | 345 +
      .../clientpositive/repl_2_exim_basic.q.out | 494 +
      .../clientpositive/repl_3_exim_metadata.q.out | 222 +
      .../runtime_skewjoin_mapjoin_spark.q.out | 692 +
      .../test/results/clientpositive/sample6.q.out | 24 +-
      .../clientpositive/script_env_var1.q.out | 8 +-
      .../clientpositive/selectDistinctStar.q.out | 92 +-
      .../clientpositive/select_dummy_source.q.out | 34 +-
      .../test/results/clientpositive/semijoin.q.out | 10 +-
      .../results/clientpositive/show_functions.q.out | 11 +
      .../results/clientpositive/show_tables.q.out | 18 +
      .../test/results/clientpositive/skewjoin.q.out | 89 +-
      .../clientpositive/skewjoin_mapjoin1.q.out | 112 +-
      .../clientpositive/skewjoin_mapjoin10.q.out | 26 +-
      .../clientpositive/skewjoin_mapjoin11.q.out | 32 +-
      .../clientpositive/skewjoin_mapjoin2.q.out | 72 +-
      .../clientpositive/skewjoin_mapjoin3.q.out | 32 +-
      .../clientpositive/skewjoin_mapjoin4.q.out | 40 +-
      .../clientpositive/skewjoin_mapjoin5.q.out | 50 +-
      .../clientpositive/skewjoin_mapjoin6.q.out | 48 +-
      .../clientpositive/skewjoin_mapjoin7.q.out | 68 +-
      .../clientpositive/skewjoin_mapjoin8.q.out | 20 +-
      .../clientpositive/skewjoin_mapjoin9.q.out | 40 +-
      .../skewjoin_union_remove_1.q.out | 144 +-
      .../skewjoin_union_remove_2.q.out | 48 +-
      .../results/clientpositive/skewjoinopt1.q.out | 144 +-
      .../results/clientpositive/skewjoinopt10.q.out | 52 +-
      .../results/clientpositive/skewjoinopt11.q.out | 84 +-
      .../results/clientpositive/skewjoinopt12.q.out | 40 +-
      .../results/clientpositive/skewjoinopt13.q.out | 28 +-
      .../results/clientpositive/skewjoinopt14.q.out | 48 +-
      .../results/clientpositive/skewjoinopt15.q.out | 34 +-
      .../results/clientpositive/skewjoinopt16.q.out | 40 +-
      .../results/clientpositive/skewjoinopt17.q.out | 80 +-
      .../results/clientpositive/skewjoinopt18.q.out | 6 +-
      .../results/clientpositive/skewjoinopt19.q.out | 40 +-
      .../results/clientpositive/skewjoinopt2.q.out | 168 +-
      .../results/clientpositive/skewjoinopt20.q.out | 40 +-
      .../results/clientpositive/skewjoinopt3.q.out | 80 +-
      .../results/clientpositive/skewjoinopt4.q.out | 80 +-
      .../results/clientpositive/skewjoinopt5.q.out | 40 +-
      .../results/clientpositive/skewjoinopt6.q.out | 40 +-
      .../results/clientpositive/skewjoinopt7.q.out | 52 +-
      .../results/clientpositive/skewjoinopt8.q.out | 52 +-
      .../results/clientpositive/skewjoinopt9.q.out | 54 +-
      .../results/clientpositive/smb_mapjoin9.q.out | 82 +
      .../results/clientpositive/smb_mapjoin_11.q.out | 1 +
      .../results/clientpositive/smb_mapjoin_12.q.out | 2 +
      .../results/clientpositive/smb_mapjoin_13.q.out | 1 +
      .../results/clientpositive/smb_mapjoin_15.q.out | 3 +
      .../clientpositive/sort_merge_join_desc_5.q.out | 1 +
      .../spark/annotate_stats_join.q.out | 10 +-
      .../clientpositive/spark/auto_join0.q.out | 97 +-
      .../clientpositive/spark/auto_join1.q.out | 8 +-
      .../clientpositive/spark/auto_join15.q.out | 2 +-
      .../clientpositive/spark/auto_join20.q.out | 4 +-
      .../clientpositive/spark/auto_join21.q.out | 2 +-
      .../clientpositive/spark/auto_join23.q.out | 2 +-
      .../clientpositive/spark/auto_join27.q.out | 7 +
      .../clientpositive/spark/auto_join28.q.out | 8 +-
      .../clientpositive/spark/auto_join29.q.out | 18 +-
      .../clientpositive/spark/auto_join30.q.out | 42 +-
      .../clientpositive/spark/auto_join31.q.out | 6 +-
      .../clientpositive/spark/auto_join32.q.out | 84 +-
      .../clientpositive/spark/auto_join8.q.out | 2 +-
      .../clientpositive/spark/auto_join_stats.q.out | 4 +-
      .../clientpositive/spark/auto_join_stats2.q.out | 8 +-
      .../spark/auto_sortmerge_join_1.q.out | 2 +
      .../spark/auto_sortmerge_join_10.q.out | 10 +
      .../spark/auto_sortmerge_join_12.q.out | 2 +-
      .../spark/auto_sortmerge_join_2.q.out | 1 +
      .../spark/auto_sortmerge_join_3.q.out | 2 +
      .../spark/auto_sortmerge_join_4.q.out | 2 +
      .../spark/auto_sortmerge_join_5.q.out | 8 +
      .../spark/auto_sortmerge_join_7.q.out | 2 +
      .../spark/auto_sortmerge_join_8.q.out | 2 +
      .../results/clientpositive/spark/bucket5.q.out | 2 +-
      .../spark/bucket_map_join_1.q.out | 10 +-
      .../spark/bucket_map_join_2.q.out | 10 +-
      .../clientpositive/spark/bucketmapjoin1.q.out | 120 +
      .../results/clientpositive/spark/cbo_gby.q.out | 124 +
      .../clientpositive/spark/cbo_limit.q.out | 90 +
      .../clientpositive/spark/cbo_semijoin.q.out | 440 +
      .../spark/cbo_simple_select.q.out | 755 +
      .../clientpositive/spark/cbo_stats.q.out | 14 +
      .../clientpositive/spark/cbo_subq_in.q.out | 149 +
      .../clientpositive/spark/cbo_subq_not_in.q.out | 365 +
      .../clientpositive/spark/cbo_udf_udaf.q.out | 125 +
      .../clientpositive/spark/cbo_union.q.out | 920 ++
      .../spark/column_access_stats.q.out | 104 +-
      .../results/clientpositive/spark/count.q.out | 10 +-
      .../spark/cross_product_check_1.q.out | 20 +-
      .../spark/cross_product_check_2.q.out | 68 +-
      .../results/clientpositive/spark/ctas.q.out | 22 +-
      .../spark/escape_clusterby1.q.out | 4 +-
      .../clientpositive/spark/escape_sortby1.q.out | 4 +-
      .../spark/filter_join_breaktask2.q.out | 8 +-
      .../clientpositive/spark/groupby10.q.out | 2 +-
      .../results/clientpositive/spark/groupby2.q.out | 8 +-
      .../results/clientpositive/spark/groupby3.q.out | 8 +-
      .../clientpositive/spark/groupby3_map.q.out | 8 +-
      .../spark/groupby3_map_multi_distinct.q.out | 8 +-
      .../spark/groupby3_map_skew.q.out | 8 +-
      .../clientpositive/spark/groupby3_noskew.q.out | 6 +-
      .../spark/groupby3_noskew_multi_distinct.q.out | 6 +-
      .../groupby7_map_multi_single_reducer.q.out | 2 +-
      .../groupby7_noskew_multi_single_reducer.q.out | 4 +-
      .../clientpositive/spark/groupby8_map.q.out | 2 +-
      .../clientpositive/spark/groupby8_noskew.q.out | 2 +-
      ...pby_complex_types_multi_single_reducer.q.out | 38 +-
      .../clientpositive/spark/groupby_cube1.q.out | 110 +-
      .../clientpositive/spark/groupby_map_ppr.q.out | 8 +-
      .../spark/groupby_map_ppr_multi_distinct.q.out | 8 +-
      .../spark/groupby_multi_single_reducer3.q.out | 8 +-
      .../clientpositive/spark/groupby_ppr.q.out | 8 +-
      .../clientpositive/spark/groupby_rollup1.q.out | 96 +-
      .../spark/groupby_sort_1_23.q.out | 19 +
      .../spark/groupby_sort_skew_1_23.q.out | 19 +
      .../spark/identity_project_remove_skip.q.out | 4 +-
      .../results/clientpositive/spark/input14.q.out | 2 +-
      .../results/clientpositive/spark/input17.q.out | 2 +-
      .../results/clientpositive/spark/input18.q.out | 2 +-
      .../results/clientpositive/spark/insert1.q.out | 46 +-
      .../clientpositive/spark/insert_into1.q.out | 92 +-
      .../clientpositive/spark/insert_into2.q.out | 88 +-
      .../clientpositive/spark/join0.q.java1.7.out | 2 +-
      .../clientpositive/spark/join0.q.java1.8.out | 238 +
      .../results/clientpositive/spark/join15.q.out | 2 +-
      .../results/clientpositive/spark/join19.q.out | 62 +-
      .../results/clientpositive/spark/join20.q.out | 4 +-
      .../results/clientpositive/spark/join21.q.out | 2 +-
      .../results/clientpositive/spark/join23.q.out | 2 +-
      .../results/clientpositive/spark/join34.q.out | 8 +
      .../results/clientpositive/spark/join35.q.out | 4 +
      .../results/clientpositive/spark/join40.q.out | 4 +-
      .../results/clientpositive/spark/join8.q.out | 2 +-
      .../spark/join_cond_pushdown_unqual1.q.out | 48 +-
      .../spark/join_cond_pushdown_unqual2.q.out | 24 +-
      .../spark/join_cond_pushdown_unqual3.q.out | 48 +-
      .../spark/join_cond_pushdown_unqual4.q.out | 24 +-
      .../clientpositive/spark/join_hive_626.q.out | 28 +-
      .../spark/join_merge_multi_expressions.q.out | 6 +-
      .../clientpositive/spark/join_nullsafe.q.out | 16 +-
      .../clientpositive/spark/join_reorder.q.out | 92 +-
      .../clientpositive/spark/join_reorder2.q.out | 68 +-
      .../clientpositive/spark/join_reorder3.q.out | 68 +-
      .../clientpositive/spark/join_reorder4.q.out | 54 +-
      .../clientpositive/spark/join_view.q.out | 32 +-
      .../spark/lateral_view_explode2.q.out | 10 +-
      .../clientpositive/spark/limit_pushdown.q.out | 106 +-
      .../spark/list_bucket_dml_2.q.java1.8.out | 665 +
      .../clientpositive/spark/load_dyn_part13.q.out | 8 +
      .../clientpositive/spark/load_dyn_part14.q.out | 17 +-
      .../clientpositive/spark/mapjoin_decimal.q.out | 49 +-
      .../spark/mapjoin_filter_on_outerjoin.q.out | 4 +-
      .../spark/mapjoin_subquery2.q.out | 30 +-
      .../spark/mapjoin_test_outer.q.out | 4 +-
      .../clientpositive/spark/mergejoins.q.out | 40 +-
      .../clientpositive/spark/mergejoins_mixed.q.out | 332 +-
      .../clientpositive/spark/multi_insert.q.out | 56 +-
      .../clientpositive/spark/multi_insert_gby.q.out | 4 +-
      .../spark/multi_insert_gby3.q.out | 6 +-
      .../spark/multi_insert_lateral_view.q.out | 2 +-
      ...i_insert_move_tasks_share_dependencies.q.out | 120 +-
      .../clientpositive/spark/multi_join_union.q.out | 8 +
      .../spark/multigroupby_singlemr.q.out | 148 +-
      .../spark/optimize_nullscan.q.out | 252 +-
      .../results/clientpositive/spark/parallel.q.out | 2 +-
      .../clientpositive/spark/parallel_join0.q.out | 2 +-
      .../clientpositive/spark/parallel_orderby.q.out | 6 +-
      .../clientpositive/spark/ppd_gby_join.q.out | 12 +-
      .../results/clientpositive/spark/ppd_join.q.out | 6 +-
      .../clientpositive/spark/ppd_join2.q.out | 81 +-
      .../clientpositive/spark/ppd_join3.q.out | 91 +-
      .../clientpositive/spark/ppd_join4.q.out | 26 +-
      .../clientpositive/spark/ppd_join5.q.out | 54 +-
      .../clientpositive/spark/ppd_outer_join1.q.out | 6 +-
      .../clientpositive/spark/ppd_outer_join2.q.out | 6 +-
      .../clientpositive/spark/ppd_outer_join4.q.out | 32 +-
      .../clientpositive/spark/ppd_outer_join5.q.out | 72 +-
      .../clientpositive/spark/ppd_transform.q.out | 4 +-
      .../test/results/clientpositive/spark/ptf.q.out | 194 +-
      .../clientpositive/spark/ptf_streaming.q.out | 100 +-
      .../spark/ql_rewrite_gbtoidx.q.out | 263 +-
      .../spark/ql_rewrite_gbtoidx_cbo_1.q.out | 263 +-
      .../spark/reduce_deduplicate.q.out | 42 +-
      .../spark/reduce_deduplicate_exclude_join.q.out | 2 +-
      .../spark/runtime_skewjoin_mapjoin_spark.q.out | 315 +
      .../results/clientpositive/spark/sample6.q.out | 24 +-
      .../clientpositive/spark/script_env_var1.q.out | 8 +-
      .../results/clientpositive/spark/semijoin.q.out | 50 +-
      .../results/clientpositive/spark/skewjoin.q.out | 87 +-
      .../spark/skewjoin_union_remove_1.q.out | 120 +-
      .../spark/skewjoin_union_remove_2.q.out | 42 +-
      .../clientpositive/spark/skewjoinopt1.q.out | 120 +-
      .../clientpositive/spark/skewjoinopt10.q.out | 32 +-
      .../clientpositive/spark/skewjoinopt11.q.out | 64 +-
      .../clientpositive/spark/skewjoinopt12.q.out | 30 +-
      .../clientpositive/spark/skewjoinopt13.q.out | 28 +-
      .../clientpositive/spark/skewjoinopt14.q.out | 40 +-
      .../clientpositive/spark/skewjoinopt15.q.out | 40 +-
      .../clientpositive/spark/skewjoinopt16.q.out | 30 +-
      .../clientpositive/spark/skewjoinopt17.q.out | 60 +-
      .../clientpositive/spark/skewjoinopt18.q.out | 6 +-
      .../clientpositive/spark/skewjoinopt19.q.out | 30 +-
      .../clientpositive/spark/skewjoinopt2.q.out | 128 +-
      .../clientpositive/spark/skewjoinopt20.q.out | 30 +-
      .../clientpositive/spark/skewjoinopt3.q.out | 60 +-
      .../clientpositive/spark/skewjoinopt4.q.out | 60 +-
      .../clientpositive/spark/skewjoinopt5.q.out | 30 +-
      .../clientpositive/spark/skewjoinopt6.q.out | 30 +-
      .../clientpositive/spark/skewjoinopt7.q.out | 42 +-
      .../clientpositive/spark/skewjoinopt8.q.out | 42 +-
      .../clientpositive/spark/skewjoinopt9.q.out | 42 +-
      .../clientpositive/spark/smb_mapjoin_11.q.out | 1 +
      .../clientpositive/spark/smb_mapjoin_12.q.out | 2 +
      .../clientpositive/spark/smb_mapjoin_13.q.out | 1 +
      .../clientpositive/spark/smb_mapjoin_15.q.out | 3 +
      .../results/clientpositive/spark/sort.q.out | 2 +-
      .../spark/sort_merge_join_desc_5.q.out | 43 +-
      .../results/clientpositive/spark/stats1.q.out | 6 +
      .../results/clientpositive/spark/stats12.q.out | 2 +-
      .../results/clientpositive/spark/stats13.q.out | 2 +-
      .../results/clientpositive/spark/stats2.q.out | 2 +-
      .../results/clientpositive/spark/stats7.q.out | 2 +-
      .../results/clientpositive/spark/stats8.q.out | 2 +-
      .../clientpositive/spark/stats_noscan_2.q.out | 8 +-
      .../clientpositive/spark/subquery_exists.q.out | 2 +
      .../clientpositive/spark/subquery_in.q.out | 8 +-
      .../spark/subquery_multiinsert.q.java1.7.out | 14 +-
      .../spark/subquery_multiinsert.q.java1.8.out | 890 ++
      .../clientpositive/spark/temp_table.q.out | 96 +-
      .../spark/tez_joins_explain.q.out | 2 -
      .../clientpositive/spark/transform1.q.out | 8 +-
      .../clientpositive/spark/transform_ppr1.q.out | 2 +-
      .../clientpositive/spark/transform_ppr2.q.out | 2 +-
      .../clientpositive/spark/udf_in_file.q.out | 8 +-
      .../results/clientpositive/spark/union.q.out | 8 +
      .../results/clientpositive/spark/union10.q.out | 12 +
      .../results/clientpositive/spark/union11.q.out | 23 +-
      .../results/clientpositive/spark/union12.q.out | 212 +
      .../results/clientpositive/spark/union13.q.out | 6 +
      .../results/clientpositive/spark/union14.q.out | 40 +-
      .../results/clientpositive/spark/union15.q.out | 45 +-
      .../results/clientpositive/spark/union16.q.out | 125 +
      .../results/clientpositive/spark/union17.q.out | 871 ++
      .../results/clientpositive/spark/union18.q.out | 10 +
      .../results/clientpositive/spark/union19.q.out | 16 +-
      .../results/clientpositive/spark/union2.q.out | 10 +
      .../results/clientpositive/spark/union20.q.out | 214 +
      .../results/clientpositive/spark/union21.q.out | 744 +
      .../results/clientpositive/spark/union22.q.out | 1611 ++
      .../results/clientpositive/spark/union23.q.out | 7 +
      .../results/clientpositive/spark/union24.q.out | 1684 ++
      .../results/clientpositive/spark/union25.q.out | 18 +
      .../results/clientpositive/spark/union26.q.out | 1265 ++
      .../results/clientpositive/spark/union27.q.out | 144 +
      .../results/clientpositive/spark/union28.q.out | 14 +
      .../results/clientpositive/spark/union29.q.out | 12 +
      .../results/clientpositive/spark/union3.q.out | 14 +-
      .../results/clientpositive/spark/union30.q.out | 18 +
      .../results/clientpositive/spark/union31.q.out | 847 +
      .../results/clientpositive/spark/union32.q.out | 699 +
      .../results/clientpositive/spark/union33.q.out | 14 +
      .../results/clientpositive/spark/union34.q.out | 425 +
      .../results/clientpositive/spark/union4.q.out | 8 +
      .../results/clientpositive/spark/union5.q.out | 16 +-
      .../results/clientpositive/spark/union6.q.out | 6 +
      .../results/clientpositive/spark/union7.q.out | 40 +-
      .../results/clientpositive/spark/union8.q.out | 9 +
      .../results/clientpositive/spark/union9.q.out | 15 +
      .../clientpositive/spark/union_date.q.out | 142 +
      .../clientpositive/spark/union_date_trim.q.out | 54 +
      .../spark/union_lateralview.q.out | 275 +
      .../clientpositive/spark/union_ppr.q.out | 12 +-
      .../clientpositive/spark/union_remove_1.q.out | 20 +-
      .../clientpositive/spark/union_remove_10.q.out | 16 +-
      .../clientpositive/spark/union_remove_11.q.out | 12 +
      .../clientpositive/spark/union_remove_12.q.out | 291 +
      .../clientpositive/spark/union_remove_13.q.out | 316 +
      .../clientpositive/spark/union_remove_14.q.out | 293 +
      .../clientpositive/spark/union_remove_15.q.out | 22 +-
      .../clientpositive/spark/union_remove_16.q.out | 22 +-
      .../clientpositive/spark/union_remove_17.q.out | 8 +
      .../clientpositive/spark/union_remove_18.q.out | 22 +-
      .../clientpositive/spark/union_remove_19.q.out | 52 +-
      .../clientpositive/spark/union_remove_2.q.out | 16 +-
      .../clientpositive/spark/union_remove_20.q.out | 22 +-
      .../clientpositive/spark/union_remove_21.q.out | 24 +-
      .../clientpositive/spark/union_remove_22.q.out | 413 +
      .../clientpositive/spark/union_remove_23.q.out | 269 +
      .../clientpositive/spark/union_remove_24.q.out | 22 +-
      .../clientpositive/spark/union_remove_25.q.out | 38 +-
      .../clientpositive/spark/union_remove_3.q.out | 12 +
      .../clientpositive/spark/union_remove_4.q.out | 20 +-
      .../clientpositive/spark/union_remove_5.q.out | 16 +-
      .../clientpositive/spark/union_remove_6.q.out | 22 +-
      .../spark/union_remove_6_subq.q.out | 1201 ++
      .../clientpositive/spark/union_remove_7.q.out | 20 +-
      .../clientpositive/spark/union_remove_8.q.out | 16 +-
      .../clientpositive/spark/union_remove_9.q.out | 18 +-
      .../clientpositive/spark/union_script.q.out | 1524 ++
      .../clientpositive/spark/union_top_level.q.out | 1054 ++
      .../clientpositive/spark/union_view.q.out | 407 +
      .../spark/vector_between_in.q.out | 6088 +-------
      .../spark/vector_cast_constant.q.java1.7.out | 16 +-
      .../spark/vector_cast_constant.q.java1.8.out | 16 +-
      .../spark/vector_count_distinct.q.out | 6 +-
      .../spark/vector_data_types.q.out | 116 +-
      .../spark/vector_decimal_mapjoin.q.out | 366 +-
      .../clientpositive/spark/vector_elt.q.out | 2 +-
      .../spark/vectorization_decimal_date.q.out | 20 +-
      .../spark/vectorization_part_project.q.out | 40 +-
      .../spark/vectorization_short_regress.q.out | 16 +-
      .../spark/vectorized_mapjoin.q.out | 8 +-
      .../spark/vectorized_nested_mapjoin.q.out | 8 +-
      .../clientpositive/spark/vectorized_ptf.q.out | 194 +-
      .../spark/vectorized_rcfile_columnar.q.out | 24 +-
      .../spark/vectorized_shufflejoin.q.out | 37 +-
      .../spark/vectorized_timestamp_funcs.q.out | 396 +-
      .../test/results/clientpositive/stats12.q.out | 2 +-
      .../test/results/clientpositive/stats13.q.out | 2 +-
      ql/src/test/results/clientpositive/stats2.q.out | 2 +-
      ql/src/test/results/clientpositive/stats7.q.out | 2 +-
      ql/src/test/results/clientpositive/stats8.q.out | 2 +-
      .../results/clientpositive/stats_noscan_2.q.out | 8 +-
      .../subq_where_serialization.q.out | 5 +
      .../clientpositive/subquery_exists.q.out | 2 +
      .../results/clientpositive/subquery_in.q.out | 8 +-
      .../subquery_in_explain_rewrite.q.out | 4 +-
      .../clientpositive/subquery_in_having.q.out | 10 +-
      .../subquery_multiinsert.q.java1.7.out | 18 +-
      .../clientpositive/subquery_notexists.q.out | 12 +-
      .../subquery_notexists_having.q.out | 12 +-
      .../results/clientpositive/subquery_notin.q.out | 74 +-
      .../subquery_notin_having.q.java1.7.out | 36 +-
      .../subquery_unqual_corr_expr.q.out | 8 +-
      .../subquery_unqualcolumnrefs.q.out | 58 +-
      .../results/clientpositive/subquery_views.q.out | 20 +-
      .../symlink_text_input_format.q.out | 24 +-
      .../clientpositive/table_nonprintable.q.out | 72 +
      .../results/clientpositive/temp_table.q.out | 84 +-
      .../temp_table_display_colstats_tbllvl.q.out | 6 +-
      .../clientpositive/temp_table_names.q.out | 8 +
      .../tez/acid_vectorization_partition.q.out | 20 +-
      .../clientpositive/tez/alter_merge_2_orc.q.out | 16 +-
      .../results/clientpositive/tez/auto_join0.q.out | 90 +-
      .../results/clientpositive/tez/auto_join1.q.out | 9 +-
      .../clientpositive/tez/auto_join29.q.out | 503 +
      .../clientpositive/tez/auto_join30.q.out | 4 +
      .../tez/auto_sortmerge_join_1.q.out | 3 +
      .../tez/auto_sortmerge_join_10.q.out | 17 +
      .../tez/auto_sortmerge_join_11.q.out | 4 +
      .../tez/auto_sortmerge_join_12.q.out | 8 +-
      .../tez/auto_sortmerge_join_13.q.out | 3 +
      .../tez/auto_sortmerge_join_14.q.out | 2 +
      .../tez/auto_sortmerge_join_15.q.out | 2 +
      .../tez/auto_sortmerge_join_2.q.out | 2 +
      .../tez/auto_sortmerge_join_3.q.out | 3 +
      .../tez/auto_sortmerge_join_4.q.out | 3 +
      .../tez/auto_sortmerge_join_5.q.out | 7 +
      .../tez/auto_sortmerge_join_7.q.out | 3 +
      .../tez/auto_sortmerge_join_8.q.out | 3 +
      .../tez/auto_sortmerge_join_9.q.out | 29 +
      .../tez/bucket_map_join_tez1.q.out | 16 +
      .../tez/bucket_map_join_tez2.q.out | 8 +
      .../results/clientpositive/tez/cbo_gby.q.out | 18 +-
      .../results/clientpositive/tez/cbo_join.q.out | 350 +-
      .../results/clientpositive/tez/cbo_limit.q.out | 4 +-
      .../clientpositive/tez/cbo_simple_select.q.out | 10 +
      .../clientpositive/tez/cbo_subq_in.q.out | 38 +-
      .../clientpositive/tez/cbo_subq_not_in.q.out | 32 +-
      .../clientpositive/tez/cbo_udf_udaf.q.out | 46 +-
      .../clientpositive/tez/cbo_windowing.q.out | 66 +-
      .../tez/correlationoptimizer1.q.out | 1 +
      .../tez/cross_product_check_1.q.out | 20 +-
      .../tez/cross_product_check_2.q.out | 97 +-
      .../test/results/clientpositive/tez/ctas.q.out | 2 +
      .../tez/dynamic_partition_pruning.q.out | 409 +-
      .../tez/dynamic_partition_pruning_2.q.out | 133 +
      .../tez/dynpart_sort_optimization2.q.out | 166 +-
      .../clientpositive/tez/explainuser_1.q.out | 1453 +-
      .../clientpositive/tez/explainuser_2.q.out | 3609 +++--
      .../tez/filter_join_breaktask2.q.out | 8 +-
      .../tez/hybridgrace_hashjoin_1.q.out | 1587 ++
      .../tez/hybridgrace_hashjoin_2.q.out | 1429 ++
      .../clientpositive/tez/hybridhashjoin.q.out | 1560 --
      .../clientpositive/tez/insert_into1.q.out | 86 +-
      .../clientpositive/tez/insert_into2.q.out | 82 +-
      .../clientpositive/tez/join_nullsafe.q.out | 20 +-
      .../clientpositive/tez/limit_pushdown.q.out | 125 +-
      .../clientpositive/tez/lvj_mapjoin.q.out | 2 +
      .../clientpositive/tez/mapjoin_decimal.q.out | 52 +-
      .../clientpositive/tez/mapjoin_mapjoin.q.out | 6 +
      .../results/clientpositive/tez/mergejoin.q.out | 3130 ++++
      .../clientpositive/tez/metadataonly1.q.out | 88 +-
      .../test/results/clientpositive/tez/mrr.q.out | 2074 +--
      .../clientpositive/tez/optimize_nullscan.q.out | 248 +-
      .../results/clientpositive/tez/orc_merge1.q.out | 8 +-
      .../test/results/clientpositive/tez/ptf.q.out | 160 +-
      .../clientpositive/tez/ptf_streaming.q.out | 84 +-
      .../clientpositive/tez/script_env_var1.q.out | 8 +-
      .../clientpositive/tez/selectDistinctStar.q.out | 108 +-
      .../results/clientpositive/tez/skewjoin.q.out | 84 +-
      .../clientpositive/tez/stats_only_null.q.out | 422 +
      .../clientpositive/tez/subquery_exists.q.out | 2 +
      .../clientpositive/tez/subquery_in.q.out | 8 +-
      .../results/clientpositive/tez/temp_table.q.out | 93 +-
      .../tez/tez_bmj_schema_evolution.q.out | 1 +
      .../results/clientpositive/tez/tez_dml.q.out | 2008 +--
      .../results/clientpositive/tez/tez_join.q.out | 28 +-
      .../clientpositive/tez/tez_join_hash.q.out | 299 +-
      .../clientpositive/tez/tez_joins_explain.q.out | 2 -
      .../clientpositive/tez/tez_multi_union.q.out | 1002 +-
      .../tez/tez_schema_evolution.q.out | 96 +-
      .../clientpositive/tez/tez_smb_main.q.out | 32 +
      .../results/clientpositive/tez/tez_union.q.out | 160 +
      .../results/clientpositive/tez/tez_union2.q.out | 18 +
      .../clientpositive/tez/tez_union_group_by.q.out | 112 +-
      .../tez/tez_union_multiinsert.q.out | 4371 ++++++
      .../results/clientpositive/tez/transform1.q.out | 8 +-
      .../results/clientpositive/tez/union2.q.out | 10 +
      .../results/clientpositive/tez/union3.q.out | 12 +
      .../results/clientpositive/tez/union4.q.out | 8 +
      .../results/clientpositive/tez/union5.q.out | 12 +-
      .../results/clientpositive/tez/union6.q.out | 6 +
      .../results/clientpositive/tez/union7.q.out | 12 +-
      .../results/clientpositive/tez/union8.q.out | 9 +
      .../results/clientpositive/tez/union9.q.out | 15 +
      .../clientpositive/tez/unionDistinct_1.q.out | 827 +-
      .../clientpositive/tez/unionDistinct_2.q.out | 38 +-
      .../results/clientpositive/tez/union_view.q.out | 1004 ++
      .../tez/update_after_multiple_inserts.q.out | 12 +-
      .../tez/update_all_partitioned.q.out | 14 +-
      .../tez/update_where_partitioned.q.out | 20 +-
      .../clientpositive/tez/vector_aggregate_9.q.out | 8 +-
      .../clientpositive/tez/vector_between_in.q.out | 6088 +-------
      .../tez/vector_binary_join_groupby.q.out | 303 +
      .../clientpositive/tez/vector_bucket.q.out | 10 +-
      .../tez/vector_char_mapjoin1.q.out | 43 +-
      .../clientpositive/tez/vector_coalesce.q.out | 26 +-
      .../tez/vector_count_distinct.q.out | 28 +-
      .../clientpositive/tez/vector_data_types.q.out | 116 +-
      .../clientpositive/tez/vector_date_1.q.out | 20 +-
      .../clientpositive/tez/vector_decimal_2.q.out | 4 +
      .../tez/vector_decimal_mapjoin.q.out | 371 +-
      .../clientpositive/tez/vector_decimal_udf.q.out | 3 +-
      .../results/clientpositive/tez/vector_elt.q.out | 2 +-
      .../tez/vector_grouping_sets.q.out | 266 +
      .../clientpositive/tez/vector_inner_join.q.out | 806 +
      .../clientpositive/tez/vector_interval_2.q.out | 60 +-
      .../tez/vector_interval_mapjoin.q.out | 281 +
      .../clientpositive/tez/vector_join30.q.out | 1367 ++
      .../tez/vector_join_filters.q.out | 222 +
      .../clientpositive/tez/vector_join_nulls.q.out | 195 +
      .../tez/vector_left_outer_join2.q.out | 555 +
      .../tez/vector_left_outer_join3.q.out | 222 +
      .../tez/vector_leftsemi_mapjoin.q.out | 13807 +++++++++++++++++
      .../tez/vector_mapjoin_reduce.q.out | 4 +
      .../tez/vector_mr_diff_schema_alias.q.out | 381 +
      .../tez/vector_null_projection.q.out | 178 +
      .../tez/vector_nullsafe_join.q.out | 1210 ++
      .../clientpositive/tez/vector_outer_join.q.out | 2204 +++
      .../clientpositive/tez/vector_outer_join0.q.out | 232 +
      .../clientpositive/tez/vector_outer_join1.q.out | 473 +
      .../clientpositive/tez/vector_outer_join2.q.out | 238 +
      .../clientpositive/tez/vector_outer_join3.q.out | 527 +
      .../clientpositive/tez/vector_outer_join4.q.out | 864 ++
      .../clientpositive/tez/vector_outer_join5.q.out | 1328 ++
      .../tez/vector_varchar_mapjoin1.q.out | 5 +-
      .../tez/vectorization_decimal_date.q.out | 20 +-
      .../tez/vectorization_limit.q.out | 31 +-
      .../tez/vectorization_part_project.q.out | 40 +-
      .../tez/vectorization_short_regress.q.out | 16 +-
      .../clientpositive/tez/vectorized_casts.q.out | 52 +-
      .../tez/vectorized_distinct_gby.q.out | 55 +-
      .../vectorized_dynamic_partition_pruning.q.out | 409 +-
      .../clientpositive/tez/vectorized_mapjoin.q.out | 9 +-
      .../tez/vectorized_nested_mapjoin.q.out | 10 +-
      .../tez/vectorized_parquet_types.q.out | 347 +
      .../clientpositive/tez/vectorized_ptf.q.out | 160 +-
      .../tez/vectorized_rcfile_columnar.q.out | 24 +-
      .../tez/vectorized_shufflejoin.q.out | 37 +-
      .../tez/vectorized_timestamp_funcs.q.out | 396 +-
      .../tez/vectorized_timestamp_ints_casts.q.out | 234 +
      .../clientpositive/timestamp_ints_casts.q.out | 264 +
      .../clientpositive/timestamp_literal.q.out | 12 +-
      .../results/clientpositive/transform1.q.out | 8 +-
      .../results/clientpositive/type_widening.q.out | 4 +-
      .../clientpositive/udaf_collect_set_2.q.out | 742 +
      .../udaf_percentile_approx_23.q.out | 6 +-
      ql/src/test/results/clientpositive/udf4.q.out | 2 +-
      ql/src/test/results/clientpositive/udf5.q.out | 30 +-
      ql/src/test/results/clientpositive/udf6.q.out | 30 +-
      ql/src/test/results/clientpositive/udf7.q.out | 2 +-
      .../results/clientpositive/udf_add_months.q.out | 4 +-
      .../clientpositive/udf_bitwise_shiftleft.q.out | 4 +-
      .../clientpositive/udf_bitwise_shiftright.q.out | 4 +-
      .../udf_bitwise_shiftrightunsigned.q.out | 4 +-
      .../test/results/clientpositive/udf_case.q.out | 2 +-
      .../test/results/clientpositive/udf_cbrt.q.out | 4 +-
      .../results/clientpositive/udf_coalesce.q.out | 4 +-
      .../test/results/clientpositive/udf_crc32.q.out | 60 +
      .../clientpositive/udf_current_database.q.out | 16 +-
      .../clientpositive/udf_date_format.q.out | 172 +
      .../results/clientpositive/udf_decode.q.out | 6 +-
      .../test/results/clientpositive/udf_elt.q.out | 4 +-
      .../results/clientpositive/udf_factorial.q.out | 4 +-
      .../clientpositive/udf_folder_constants.q.out | 120 +
      .../clientpositive/udf_from_utc_timestamp.q.out | 4 +-
      .../results/clientpositive/udf_greatest.q.out | 4 +-
      .../test/results/clientpositive/udf_hour.q.out | 2 +-
      ql/src/test/results/clientpositive/udf_if.q.out | 4 +-
      .../results/clientpositive/udf_in_file.q.out | 10 +-
      .../test/results/clientpositive/udf_instr.q.out | 4 +-
      .../clientpositive/udf_isnull_isnotnull.q.out | 2 +-
      .../results/clientpositive/udf_last_day.q.out | 4 +-
      .../test/results/clientpositive/udf_least.q.out | 4 +-
      .../results/clientpositive/udf_length.q.out | 4 +-
      .../clientpositive/udf_levenshtein.q.out | 4 +-
      .../results/clientpositive/udf_locate.q.out | 4 +-
      .../test/results/clientpositive/udf_md5.q.out | 61 +
      .../results/clientpositive/udf_minute.q.out | 2 +-
      .../clientpositive/udf_months_between.q.out | 4 +-
      .../test/results/clientpositive/udf_nvl.q.out | 2 +-
      .../results/clientpositive/udf_parse_url.q.out | 2 +-
      .../results/clientpositive/udf_quarter.q.out | 246 +
      .../results/clientpositive/udf_second.q.out | 2 +-
      .../test/results/clientpositive/udf_sha1.q.out | 61 +
      .../test/results/clientpositive/udf_sha2.q.out | 134 +
      .../test/results/clientpositive/udf_size.q.out | 2 +-
      .../results/clientpositive/udf_sort_array.q.out | 33 +
      .../results/clientpositive/udf_soundex.q.out | 4 +-
      .../clientpositive/udf_substring_index.q.out | 107 +
      .../clientpositive/udf_to_utc_timestamp.q.out | 4 +-
      .../test/results/clientpositive/udf_trunc.q.out | 88 +-
      .../clientpositive/udf_unix_timestamp.q.out | 32 +-
      .../test/results/clientpositive/udf_when.q.out | 4 +-
      .../results/clientpositive/udtf_stack.q.out | 8 +-
      .../test/results/clientpositive/union10.q.out | 12 +-
      .../test/results/clientpositive/union11.q.out | 16 +-
      .../test/results/clientpositive/union12.q.out | 20 +-
      .../test/results/clientpositive/union14.q.out | 12 +-
      .../test/results/clientpositive/union15.q.out | 16 +-
      .../test/results/clientpositive/union17.q.out | 26 +-
      .../test/results/clientpositive/union18.q.out | 8 +-
      .../test/results/clientpositive/union19.q.out | 20 +-
      .../test/results/clientpositive/union20.q.out | 26 +-
      .../test/results/clientpositive/union21.q.out | 30 +-
      .../test/results/clientpositive/union22.q.out | 7 +
      .../test/results/clientpositive/union27.q.out | 4 +-
      .../test/results/clientpositive/union30.q.out | 42 +-
      ql/src/test/results/clientpositive/union4.q.out | 8 +-
      ql/src/test/results/clientpositive/union5.q.out | 12 +-
      ql/src/test/results/clientpositive/union7.q.out | 12 +-
      .../clientpositive/unionDistinct_1.q.out | 351 +-
      .../clientpositive/unionDistinct_2.q.out | 42 +-
      .../clientpositive/union_lateralview.q.out | 16 +-
      .../results/clientpositive/union_remove_1.q.out | 24 +-
      .../clientpositive/union_remove_10.q.out | 24 +-
      .../clientpositive/union_remove_11.q.out | 30 +-
      .../clientpositive/union_remove_12.q.out | 20 +-
      .../clientpositive/union_remove_13.q.out | 28 +-
      .../clientpositive/union_remove_14.q.out | 20 +-
      .../clientpositive/union_remove_15.q.out | 28 +-
      .../clientpositive/union_remove_16.q.out | 28 +-
      .../clientpositive/union_remove_17.q.out | 20 +-
      .../clientpositive/union_remove_18.q.out | 28 +-
      .../clientpositive/union_remove_19.q.out | 56 +-
      .../results/clientpositive/union_remove_2.q.out | 24 +-
      .../clientpositive/union_remove_20.q.out | 28 +-
      .../clientpositive/union_remove_21.q.out | 24 +-
      .../clientpositive/union_remove_22.q.out | 72 +-
      .../clientpositive/union_remove_23.q.out | 34 +-
      .../clientpositive/union_remove_24.q.out | 28 +-
      .../clientpositive/union_remove_25.q.out | 24 +-
      .../results/clientpositive/union_remove_3.q.out | 30 +-
      .../results/clientpositive/union_remove_4.q.out | 24 +-
      .../results/clientpositive/union_remove_5.q.out | 24 +-
      .../results/clientpositive/union_remove_6.q.out | 32 +-
      .../clientpositive/union_remove_6_subq.q.out | 44 +-
      .../results/clientpositive/union_remove_7.q.out | 24 +-
      .../results/clientpositive/union_remove_8.q.out | 24 +-
      .../results/clientpositive/union_remove_9.q.out | 32 +-
      .../results/clientpositive/union_view.q.out | 408 +-
      .../update_after_multiple_inserts.q.out | 12 +-
      .../clientpositive/update_all_partitioned.q.out | 14 +-
      .../update_where_partitioned.q.out | 20 +-
      .../clientpositive/vector_aggregate_9.q.out | 8 +-
      .../clientpositive/vector_between_in.q.out | 6088 +-------
      .../vector_binary_join_groupby.q.out | 293 +
      .../results/clientpositive/vector_bucket.q.out | 10 +-
      .../clientpositive/vector_char_mapjoin1.q.out | 36 +-
      .../clientpositive/vector_coalesce.q.out | 6 +-
      .../clientpositive/vector_data_types.q.out | 116 +-
      .../results/clientpositive/vector_date_1.q.out | 20 +-
      .../clientpositive/vector_decimal_2.q.out | 4 +
      .../clientpositive/vector_decimal_mapjoin.q.out | 370 +-
      .../clientpositive/vector_decimal_udf.q.out | 3 +-
      .../results/clientpositive/vector_elt.q.out | 2 +-
      .../clientpositive/vector_grouping_sets.q.out | 253 +
      .../clientpositive/vector_inner_join.q.out | 799 +
      .../clientpositive/vector_interval_2.q.out | 60 +-
      .../vector_interval_mapjoin.q.out | 281 +
      .../results/clientpositive/vector_join30.q.out | 2218 +++
      .../clientpositive/vector_join_filters.q.out | 222 +
      .../clientpositive/vector_join_nulls.q.out | 195 +
      .../vector_left_outer_join2.q.out | 572 +
      .../vector_leftsemi_mapjoin.q.out | 13572 ++++++++++++++++
      .../clientpositive/vector_mapjoin_reduce.q.out | 12 +
      .../vector_mr_diff_schema_alias.q.out | 396 +
      .../clientpositive/vector_multi_insert.q.out | 6 +-
      .../clientpositive/vector_null_projection.q.out | 167 +
      .../clientpositive/vector_nullsafe_join.q.out | 1208 ++
      .../clientpositive/vector_outer_join0.q.out | 230 +
      .../clientpositive/vector_outer_join1.q.out | 466 +
      .../clientpositive/vector_outer_join2.q.out | 232 +
      .../clientpositive/vector_outer_join3.q.out | 509 +
      .../clientpositive/vector_outer_join4.q.out | 857 +
      .../clientpositive/vector_outer_join5.q.out | 1300 ++
      .../vector_varchar_mapjoin1.q.out | 4 +-
      .../vectorization_decimal_date.q.out | 20 +-
      .../vectorization_part_project.q.out | 40 +-
      .../vectorization_short_regress.q.out | 16 +-
      .../clientpositive/vectorized_casts.q.out | 52 +-
      .../vectorized_distinct_gby.q.out | 4 +-
      .../clientpositive/vectorized_mapjoin.q.out | 8 +-
      .../vectorized_nested_mapjoin.q.out | 8 +-
      .../results/clientpositive/vectorized_ptf.q.out | 167 +-
      .../vectorized_rcfile_columnar.q.out | 24 +-
      .../clientpositive/vectorized_shufflejoin.q.out | 45 +-
      .../vectorized_timestamp_funcs.q.out | 396 +-
      .../vectorized_timestamp_ints_casts.q.out | 266 +
      .../clientpositive/windowing_navfn.q.out | 10 +
      .../clientpositive/windowing_streaming.q.out | 16 +-
      .../clientpositive/windowing_windowspec2.q.out | 3084 ++++
      ql/src/test/templates/TestCliDriver.vm | 18 +-
      ql/src/test/templates/TestCompareCliDriver.vm | 21 +-
      ql/src/test/templates/TestNegativeCliDriver.vm | 18 +-
      ql/src/test/templates/TestParseNegative.vm | 17 +-
      serde/pom.xml | 2 +-
      .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 19 +-
      .../hadoop/hive/serde2/NullStructSerDe.java | 2 +-
      .../apache/hadoop/hive/serde2/SerDeUtils.java | 1 +
      .../apache/hadoop/hive/serde2/WriteBuffers.java | 25 +-
      .../binarysortable/BinarySortableSerDe.java | 157 +-
      .../fast/BinarySortableSerializeWrite.java | 102 +-
      .../hadoop/hive/serde2/fast/SerializeWrite.java | 12 +-
      .../hive/serde2/io/HiveDecimalWritable.java | 5 +-
      .../hive/serde2/io/HiveVarcharWritable.java | 7 +-
      .../hive/serde2/io/TimestampWritable.java | 11 +-
      .../hive/serde2/lazy/LazySerDeParameters.java | 18 +
      .../lazy/fast/LazySimpleSerializeWrite.java | 13 +-
      .../LazyObjectInspectorFactory.java | 24 +-
      .../LazyPrimitiveObjectInspectorFactory.java | 41 +-
      .../hive/serde2/lazybinary/LazyBinaryUtils.java | 19 +-
      .../fast/LazyBinarySerializeWrite.java | 15 +-
      .../LazyBinaryObjectInspectorFactory.java | 24 +-
      .../MetadataListStructObjectInspector.java | 18 +-
      .../ObjectInspectorConverters.java | 5 +-
      .../objectinspector/ObjectInspectorUtils.java | 6 +-
      .../AbstractPrimitiveObjectInspector.java | 1 +
      .../PrimitiveObjectInspectorConverter.java | 7 +-
      .../PrimitiveObjectInspectorFactory.java | 26 +-
      .../PrimitiveObjectInspectorUtils.java | 42 +-
      .../primitive/WritableVoidObjectInspector.java | 5 +
      .../hive/serde2/typeinfo/CharTypeInfo.java | 5 +-
      .../hive/serde2/typeinfo/DecimalTypeInfo.java | 7 +-
      .../hive/serde2/typeinfo/PrimitiveTypeInfo.java | 5 +-
      .../hive/serde2/typeinfo/TypeInfoFactory.java | 31 +-
      .../hive/serde2/typeinfo/TypeInfoUtils.java | 27 +-
      .../hive/serde2/typeinfo/VarcharTypeInfo.java | 5 +-
      .../apache/hadoop/hive/serde2/VerifyFast.java | 10 +-
      .../hive/serde2/binarysortable/MyTestClass.java | 105 +
      .../binarysortable/MyTestPrimitiveClass.java | 20 +
      .../binarysortable/TestBinarySortableFast.java | 13 +-
      .../binarysortable/TestBinarySortableSerDe.java | 10 +-
      .../hive/serde2/lazy/TestLazySimpleFast.java | 22 +-
      .../hive/serde2/lazy/TestLazySimpleSerDe.java | 9 +-
      .../TestPrimitiveObjectInspectorUtils.java | 95 +-
      service/pom.xml | 7 +-
      .../org/apache/hive/service/CookieSigner.java | 108 +
      .../apache/hive/service/auth/HttpAuthUtils.java | 72 +
      .../auth/LdapAuthenticationProviderImpl.java | 2 +-
      .../apache/hive/service/cli/ColumnValue.java | 2 +
      .../java/org/apache/hive/service/cli/Type.java | 2 +-
      .../cli/operation/GetCatalogsOperation.java | 12 +-
      .../cli/operation/GetColumnsOperation.java | 34 +
      .../cli/operation/GetFunctionsOperation.java | 25 +
      .../cli/operation/GetSchemasOperation.java | 13 +
      .../cli/operation/GetTableTypesOperation.java | 4 +
      .../cli/operation/GetTablesOperation.java | 10 +
      .../cli/operation/GetTypeInfoOperation.java | 4 +
      .../cli/operation/MetadataOperation.java | 35 +
      .../service/cli/operation/OperationManager.java | 13 +-
      .../service/cli/session/HiveSessionImpl.java | 16 +-
      .../service/cli/session/SessionManager.java | 49 +-
      .../thrift/EmbeddedThriftBinaryCLIService.java | 5 +
      .../cli/thrift/ThriftBinaryCLIService.java | 1 +
      .../service/cli/thrift/ThriftCLIService.java | 79 +-
      .../service/cli/thrift/ThriftHttpServlet.java | 212 +-
      .../apache/hive/service/server/HiveServer2.java | 161 +-
      .../apache/hive/service/TestCookieSigner.java | 59 +
      .../TestLdapAuthenticationProviderImpl.java | 18 +
      .../service/cli/session/TestSessionHooks.java | 3 +-
      .../thrift/ThriftCliServiceTestWithCookie.java | 221 +
      shims/0.20S/pom.xml | 2 +-
      .../hadoop/hive/shims/Hadoop20SShims.java | 30 +
      shims/0.23/pom.xml | 2 +-
      .../apache/hadoop/hive/shims/Hadoop23Shims.java | 40 +-
      shims/aggregator/pom.xml | 2 +-
      shims/common/pom.xml | 2 +-
      .../apache/hadoop/hive/shims/HadoopShims.java | 20 +
      .../hadoop/hive/shims/HadoopShimsSecure.java | 4 +
      shims/pom.xml | 2 +-
      shims/scheduler/pom.xml | 2 +-
      spark-client/pom.xml | 4 +-
      .../apache/hive/spark/client/JobContext.java | 8 +-
      .../hive/spark/client/JobContextImpl.java | 17 +-
      .../apache/hive/spark/client/RemoteDriver.java | 34 +-
      .../hive/spark/client/SparkClientImpl.java | 39 +-
      .../hive/spark/client/SparkClientUtilities.java | 28 +-
      .../apache/hive/spark/client/rpc/RpcServer.java | 29 +-
      .../apache/hive/spark/client/rpc/TestRpc.java | 32 +-
      testutils/metastore/dbs/derby/execute.sh | 37 +
      testutils/metastore/dbs/derby/prepare.sh | 63 +
      testutils/metastore/dbs/postgres/execute.sh | 29 +
      testutils/metastore/dbs/postgres/prepare.sh | 72 +
      testutils/metastore/execute-test-on-lxc.sh | 47 +-
      testutils/metastore/metastore-upgrade-test.sh | 5 +-
      testutils/pom.xml | 2 +-
      .../org/apache/hive/ptest/execution/PTest.java | 12 +-
      .../ptest/execution/conf/TestConfiguration.java | 12 +-
      .../ptest2/src/main/resources/source-prep.vm | 11 +
      .../hive/ptest/execution/TestScripts.java | 23 +
      .../TestScripts.testPrepGit.approved.txt | 2 +-
      .../TestScripts.testPrepHadoop1.approved.txt | 111 +
      .../TestScripts.testPrepNone.approved.txt | 2 +-
      .../TestScripts.testPrepSvn.approved.txt | 2 +-
      .../execution/conf/TestTestConfiguration.java | 38 +-
      .../resources/test-configuration.properties | 2 +
      2000 files changed, 188566 insertions(+), 45392 deletions(-)
    ----------------------------------------------------------------------
  • Gates at Jun 5, 2015 at 11:16 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/itests/hive-unit/pom.xml
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/if/hive_metastore.thrift
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/pom.xml
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/pom.xml
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    ----------------------------------------------------------------------

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a64664a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
    ----------------------------------------------------------------------
  • Gates at Jun 5, 2015 at 11:16 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/data/files/vectortab_b_1k
    ----------------------------------------------------------------------
    diff --git a/data/files/vectortab_b_1k b/data/files/vectortab_b_1k
    new file mode 100644
    index 0000000..812d98a
    --- /dev/null
    +++ b/data/files/vectortab_b_1k
    @@ -0,0 +1,1000 @@
    +38|28413|-469870330|7061809776248545280|5038.36|4899225.93||true|american history|wendy van buren|2052-03-21 12:31:39|2041-01-06 14:03:47.179206937|2074-03-12
    +108|-5135|-2042831105|6969599299897163776|-40691.70|-1564446.85|-4298929701839.924316|false|philosophy||2033-10-13 05:36:52|2014-06-18 16:42:39.89307305|2079-04-19
    +-34|11168|1846184880|-7535857766791577600|-12097.91|296884.75|189835086132.674805|false|wind surfing|zach ellison|2022-01-21 10:58:22|2014-08-14 21:48:41.258440751|2058-03-24
    +42|-7323|1625751062|3724|19175.45|-4645302.81||true|yard duty|jessica nixon|2046-12-23 10:11:15|2040-05-12 13:49:04.312695207|2067-12-19
    +-101|20084|343362793|2786|-5759.71|-3710545.39|-747309143932.177734|false|xylophone band|nick johnson|2055-11-22 06:32:07|2032-12-27 00:39:24.505679054|2013-11-15
    +33|-18263|735732067|1948|38213.05|2668590.35|915569735455.541016|true|quiet hour|rachel ovid|2018-03-02 05:25:36|2063-07-27 10:41:50.110054963|2046-11-16
    +-54|-21795|-1017629298|-7328087811698909184|-20010.24|-1103528.09|171257816833.722656|true|geology|wendy brown|2022-12-25 05:14:20|2069-06-10 23:30:18.814929160|2078-03-20
    +16|-14888|1190302173|7753882935005880320|-13338.98|-3830938.65|-16966857031.200195|false|kindergarten|priscilla ichabod|2079-09-20 10:22:11|2066-05-11 11:44:44.967158232|2058-05-02
    +-104|22213|-1818456584|-8384695077413412864|43493.35|-828620.69|2303273645231.165039|true|nap time|fred garcia||2058-03-20 03:18:32.441094681|2005-09-08
    +76|-26087|-295186284|-9075302542655684608|-16572.92|2456302.57|1695145412960.172852|true|mathematics|rachel thompson|2067-11-29 10:49:14|2023-02-24 06:11:31.127903962|1970-10-04
    +-105|-4953|315973457|-8844949406948671488|-9860.94|998897.88|-4775905812492.879883|false|education|fred davidson|2025-09-30 18:11:16|2025-06-11 16:43:58.941547150|1999-12-25
    +-127|16110|904604938|8337549596011102208|2848.64|-3418610.14|-2009642318242.228516|true|debate|mike steinbeck|2023-04-26 20:45:39|2048-06-21 02:28:33.302451076|2068-01-05
    +-109|31986|-1218581850|-7624057992767782912|-9642.07||4524736841293.623047|true|chemistry|ethan ovid|2046-01-30 17:53:58|2064-06-05 07:51:39.21714322|2097-09-03
    +116|-32119|-2144241640|9073672806863790080|-7691.49|-4810810.06|-3529351191949.139648|true|zync studies|quinn ichabod|2025-09-14 05:57:47|2052-08-21 07:01:03.126893909|2101-08-31
    +-69|-28864|1766517223|7948803266578161664|-33985.35|3550700.26|4086092318223.570312|false|debate||2071-01-12 01:01:39|2059-09-08 02:39:24.970547582|1989-11-05
    +-13|12814|881673558|7566273236152721408|-4594.45|-3286472.17|423483325042.503906|false|linguistics|nick underhill|2023-06-25 22:14:52||2064-02-13
    +4|-10513|536876888|7062382339142156288|13262.02|4083053.68||false||wendy young|2073-07-01 02:10:21|2068-11-13 07:56:42.534738020|2026-07-12
    +-109|-20188|-1289665817|7909645665163804672|42529.03|2938682.72|4492997493634.832031|false|values clariffication||2036-02-03 11:35:01|2016-09-22 08:56:11.169028039|2065-08-11
    +-33|20120|-340462064|-8086577583338061824|30435.53|4916056.68|2409115470740.985352|true|zync studies|calvin quirinius|2052-08-12 00:26:09|2019-08-25 23:11:02.671444276|1991-04-05
    +-82|-23630|1425456189|-8122639684164501504|13589.93|-4405035.26|3998360085714.693359|true|geology||2055-05-29 17:37:08||2006-11-30
    +-18|-19786|-1642207005|-8117838333114212352|17411.05||1866266907288.434570|false|wind surfing|sarah garcia|2028-01-29 13:44:26|2039-09-01 15:14:46.549821420|1975-12-21
    +-83|23910|1376818328|336|-27132.88|1204353.22|2806463824394.356445|true|undecided|sarah ellison|2080-10-30 17:04:15|2065-12-02 08:54:56.954538295|2014-02-02
    +111|32231|-1919939921|2320|-6568.58|1215239.64|-1869700480167.893066|true|kindergarten|xavier van buren||2016-07-10 10:53:59.626066496|2021-05-21
    +-86|12915|-1434562279|-8731068123910987776|-8835.22|-1335137.34|-1538575730095.495117|false|american history||2061-12-03 00:29:54|2072-08-27 13:08:21.852423838|2016-07-12
    +-107|-13|1070989126|1906|-12761.32|3064168.48|-3462773458406.651367|true||fred carson|2018-12-15 02:28:19|2071-03-20 11:13:17.416475043|2018-06-15
    +95|-20789|1028204648|-7838598833900584960|1716.08|-2554550.12|4714480698801.324219|true|geology|bob falkner|2045-07-31 00:13:55|2059-03-29 14:57:28.222857121|2099-08-04
    +-80|-8852|477857533|1165|37201.95||-1261362806924.886230|true|forestry|jessica ovid|2062-04-27 20:14:10||2064-08-16
    +|-14783|-224865887|-7456869587112255488|5122.61|-2102976.63|1531516008788.393555|false|values clariffication|fred nixon|2048-10-12 23:56:55|2026-06-07 14:49:02.306537960|2089-05-15
    +-15|17581|1142098316|2013|6003.12|-1151629.14|4348345225726.326172|false|history|yuri van buren|2024-12-02 01:39:53|2073-03-07 20:51:20.326376818|2050-01-09
    +-8|-24885|-332125121|7333512171174223872|26535.92|2399386.54|-2960944140681.114258|false|opthamology|ethan white|2079-02-15 19:05:52||2087-01-27
    +-89|-10935|1240875512|-7571293705217687552|21259.00|-629766.26|-1636595770961.833008|false|american history|david quirinius|2033-01-07 17:02:27|2063-10-15 04:11:02.388323396|2096-08-29
    +-19|6306|-590374062|-8523434203900674048|28590.02|1265094.91|3833331173378.199219|true|undecided|victor thompson|2056-03-02 15:07:50|2047-02-04 15:17:32.647082767|1990-11-23
    +-39|-16680|492968645|8099215208813903872|11064.77|750795.33|-3997116358934.107422|false|zync studies|sarah underhill|2032-12-13 09:50:46|2029-10-11 21:31:15.496921510|2001-01-25
    +-48|-30157|-1635301453|9040958359122640896|42146.27|-1984020.04|-3919914678779.932129|false|yard duty|luke ichabod|2050-05-21 14:47:18|2072-06-27 06:27:52.850736616|2012-07-08
    +118|-21389|1319589591|-7356685674003021824|48287.81|4413852.26|4754891500688.814453|false|philosophy|||2031-07-02 15:06:48.138419705|2092-03-31
    +-118|13138|-1652600376|2072|31787.11|4413364.49||false|opthamology|victor zipper||2058-07-13 04:38:16.443396189|1996-08-31
    +32|29023|-856843296|2073|48554.24|-2764932.99|4532387406205.238281|false|mathematics|jessica polk|2066-09-16 03:57:39|2065-06-11 16:20:38.164287829|2011-01-20
    +-31|-9496|915505006|871|37794.69|4782774.03|3414437080001.526367|false|geology|luke hernandez|2061-05-20 02:26:59|2068-01-21 09:27:30.854734737|2023-12-26
    +22|-6460|1179528290|-7551394356730339328|-23041.73|-4296206.08|4962408770099.849609|true|xylophone band|luke falkner|2058-10-05 19:19:16||2021-12-19
    +83|-29675|-113253627|-8172827216441573376|-1794.84|-317220.62|688329351482.579102|true|biology|david robinson|2039-03-30 18:42:28|2020-02-03 05:20:18.744684653|2050-09-20
    +88|6761|-1878838836|-8082793390939193344|36116.48|1298897.65|1744469246864.212891|false|chemistry|tom ellison|2030-01-25 11:28:31|2016-05-22 14:39:37.64576689|2044-06-20
    +-49|-12588|2065408093|8854495099223375872|25810.56||-3371566685461.750977|false|opthamology|priscilla polk|2029-08-07 14:29:01|2048-03-29 02:01:01.562257980|2000-08-18
    +-14|28008|-122391516|-8358130693961195520|-33737.35|4303885.61|207192707064.258789|true|joggying|alice garcia|2025-11-21 05:44:38|2029-03-08 07:36:34.632835852|2105-01-06
    +-52|27527|-1240048334|9050032047355125760|-41056.39|-2126298.83|3424231918266.365234|false|education|sarah ellison|2055-05-29 10:01:01|2053-06-19 04:48:57.10910076|2004-06-03
    +84||1813010930|-7162299524557471744|-4970.18|910733.08|-4747435156455.921875|false|history|quinn ovid|2056-11-24 23:44:36|2054-10-30 00:49:45.287291218|2070-02-03
    +28|-21506|-682333536|809|-13948.95|2291401.51|33266358053.083008|false|geology|ulysses steinbeck|2080-06-19 15:17:59||1984-12-28
    +4||-759911896|-8946656952763777024|-4183.99|-4038048.77|-3167988150233.824707|false|opthamology|katie white|2061-09-28 17:08:58|2015-09-01 23:29:55.301806098|2032-01-18
    +-100|13491|-1369302744|1053|-10809.39|4440019.40|-1117844143988.975098||values clariffication|quinn thompson|2023-03-09 21:06:33|2027-12-13 08:22:59.973086123|2015-09-29
    +4||765084282|482|-24177.64|-3926632.94|-128454528929.609375||biology|ulysses johnson|2039-12-01 00:28:17|2047-02-12 11:40:45.586393217|2090-06-30
    +126|-16307|470993066|-6968892545529896960|2416.88|-767525.41|-4599746514511.089844|true|industrial engineering|luke garcia||2068-09-30 15:34:57.912726771|2043-08-23
    +-3|24819|2070969353|203|49013.15|3648039.90|-2530166692170.396973|false|religion|zach davidson|2028-03-11 20:18:02|2023-01-31 06:26:26.819065415|2026-12-24
    +-79|-19571|-296195507|1614|1339.61|472805.29|-4628165921333.362305|false|geology|victor thompson|2025-05-07 23:33:22|2076-12-14 19:01:33.673146196|2040-05-30
    +88|7057|6266567|-8593419958317056000|20993.37|3470920.76|2907965923466.875000|false|opthamology|jessica van buren|2026-04-10 03:58:42|2027-05-24 20:24:57.709957719|1985-01-18
    +42|-562|604460005|8190967051000659968|10778.57|1505668.86|-4298212148464.983887|false|american history|priscilla king|2071-01-07 17:48:53|2050-08-28 10:51:32.79079993|2036-07-23
    +-5|4536|-1836166334|808|-26348.93|1609583.36|-2450581027868.068359|false|biology|zach ellison|2064-03-18 16:36:24|2040-12-19 14:19:07.29019956|2094-02-23
    +31|2855|458910170||4194.56|-2703308.04|-3920065100013.161621|false|geology|holly ellison||2064-10-06 03:04:54.202018334|2058-01-16
    +-36|813|2144454927|412|-17894.49|3767875.02|-4076956189315.090820|true|zync studies|irene young|2051-03-22 13:42:47||2069-11-02
    +-19|-16002||8656571350884048896|44168.85|4141077.81|2689794991277.487305|true|yard duty|rachel steinbeck|2038-02-19 06:53:09|2052-02-15 19:04:26.991933690|2018-01-25
    +-123|25732|2100377172|8769199243315814400|14038.74|191532.01|-3373496595819.993652|false|quiet hour|quinn allen|2080-05-30 20:51:34|2038-06-11 05:22:17.584841494|1997-09-20
    +-28|-9278|-1236536142|-8546758906409312256|-18615.91|-385172.08|3145518148645.542969|false|values clariffication|quinn ellison||2050-08-26 21:46:55.776228072|2006-05-04
    +90|-27844||8829545979081744384|-7890.36|-4687872.68|-189051209331.019531|true|forestry|holly ellison|2078-01-22 08:05:44|2051-12-30 01:07:06.843556116|2019-10-31
    +-33|6756|-1380678829|7545689659010949120|3119.59|1219503.88|1383475996151.766602|true|chemistry|gabriella nixon|2013-11-15 10:53:04|2074-01-13 06:13:44.309707032|2086-07-20
    +-27|32063|-412333994|618|18352.29|-2682908.49|-2196296807639.192871|false|xylophone band|mike ichabod|2049-05-17 01:36:05|2016-01-06 04:27:12.937792140|1988-07-21
    +115|14089|1583280136|8573305425181941760|-12596.11|-4306107.96|3399085059351.882812|true|forestry|bob carson|2077-10-22 01:28:01|2070-11-22 05:00:18.148513417|2060-02-01
    +87|-5837||94|31117.44|4356549.19|3648188962203.846680|false|joggying|quinn nixon|2013-11-03 06:51:21|2060-06-22 12:16:37.962088375|2101-04-20
    +42|6036|960187615|-7266719102957125632|-47893.89|-295854.59|2917455244518.621094|false|topology|calvin zipper||2064-05-28 07:51:38.18135161|2091-10-08
    +-57|-16290|1835749815|2772|-2499.18|-2208789.27|-1579859560936.931641|true|joggying|wendy carson|2068-08-26 13:27:51|2021-11-05 06:16:08.956633171|2012-03-06
    +34|-2518|1625699061|379|-24309.03|3375470.42|-1718131278688.766113|false|education|ethan johnson|2046-02-23 07:23:16|2040-11-20 23:57:22.271546670|1978-07-29
    +69|28358|-1524081566|8302473563519950848|-297.67|-3373283.08|-2009241907914.223145|true|joggying|ethan van buren|2047-08-15 06:48:02|2043-04-13 19:11:51.590383118|
    +-11|31334|215759857|-7802538500225777664|33944.93|-2034061.98|-3034548052198.289062|false|industrial engineering|sarah steinbeck|2055-02-11 13:24:46|2053-11-27 08:12:32.905325041|2039-02-11
    +76|-25155|2009890220|-7273694358642851840|18007.61|-1882990.85|-1586767512900.379883|false|religion|nick underhill|2043-09-05 07:30:29|2039-05-08 08:23:28.744105983|2012-06-13
    +-42|-27015|-1024500955|8987827141270880256|6071.06|||false|history|rachel thompson|2080-11-04 06:42:28|2022-05-07 19:23:26.462479493|1990-08-27
    +-91|-7300|-1257859205|914|-5866.74|-1809849.07|-1670602067707.562988|true|xylophone band|mike garcia|2057-06-04 13:48:28|2056-12-01 04:07:20.17327137|2090-07-13
    +|-13972|1616782308|723|15887.15|-2469559.45|-380705728848.234375|false|american history|jessica quirinius|2057-08-11 06:45:24||2003-06-16
    +-69|-10569|-1928197479|-7104310188119834624|26076.44|2535185.38|2312022922718.455078|false|linguistics|calvin falkner|2033-03-03 06:25:26|2015-06-15 18:44:04.445242368|2020-04-30
    +-101|12583|998793176|154|-39236.48|2570101.79|-2715354441339.101562|false|history|xavier falkner|2062-06-17 13:37:19|2065-12-13 13:46:25.594101554|1983-02-08
    +28|25518|631207613|-8494118409594650624|33125.44|-315524.15|4237441963242.361328|false|study skills|tom garcia|2048-09-29 05:52:03|2066-12-04 01:20:15.241688388|1988-03-29
    +-40|-31164|-161884324|3781|-49606.91|1038717.47|155199366725.398438|false|joggying|oscar nixon|2023-10-07 10:39:24|2071-02-16 02:10:47.198374359|2055-05-12
    +-124|-5267|574069547|1466|4445.84|3318110.58|3262707195109.015625|true|industrial engineering|tom falkner|2049-01-22 07:20:24|2013-04-19 01:40:57.733559293|2064-01-22
    +-28|-20663|-616724730|724|43976.01|-254306.16|-2564468363230.716309|true|industrial engineering|mike brown|2040-06-27 02:12:07|2027-09-24 00:10:50.868223193|2043-01-07
    +|2542|-1984079412|-7270034223527993344|42132.29|1872487.12|4239681407336.525391|true|undecided|holly ichabod|2036-02-28 11:43:29|2058-09-07 02:46:35.228498480|2052-03-13
    +-62|-25077|1845797092|913|18763.32|622185.07|2205467425785.875977|false|religion|alice zipper|2072-02-01 03:42:53|2054-05-21 06:31:30.994303844|2024-02-23
    +64|32309|-605370177|1704|46536.51|-2429187.29|-3869444972623.164062|false|linguistics|priscilla davidson|2047-07-10 07:46:00|2047-03-10 19:56:43.16082472|2077-03-07
    +-21|2590|-915104901||-19995.58|2053186.77|957236234850.747070|false|wind surfing|yuri carson|2015-06-30 01:25:11|2070-03-17 03:33:19.30902666|2036-02-16
    +-75|-12782|1136548971|-7883252982752665600|-814.76|2576885.92|2555233457696.850586|false|geology||2024-05-25 02:43:22|2040-11-17 01:26:44.81314350|2000-03-04
    +127|-27312|-505879576|412|39913.52|2317989.11|4849612269862.681641|false|opthamology|holly steinbeck|2048-01-19 12:38:07|2054-11-08 01:30:13.939944047|2000-05-28
    +-28|-25412|-202035134|4024|-4465.40|-1742902.49|4667839539207.853516|true|undecided|katie underhill|2039-09-02 21:29:19|2071-08-10 05:34:49.654645949|2098-09-08
    +-26||-935723237|7226360892091416576|-31695.38|2802498.83|2390739186802.050781|true|kindergarten||2051-11-30 06:15:36|2050-03-18 13:45:32.683739800|2052-09-26
    +-115|-28939|688547276|-8244657976255889408|36162.80|-4989122.80|-2552765883743.006836|true|xylophone band||2034-11-26 00:25:11||2025-05-13
    +-86|-12071|1956887369|2862|1570.24|-4626990.97|3666109195511.175781|false||bob king|2051-01-24 01:38:45|2055-03-11 20:34:57.197479653|2034-02-08
    +-67|-1443|-993029335|1521|-46155.68|4524183.68||false|debate|oscar laertes|2058-04-21 07:57:00|2078-08-31 09:56:45.671467607|1982-04-01
    +7|7530|-549167249||20406.25|1525395.61|-1198863806969.718750|false|history|holly nixon|2043-01-28 08:16:12|2045-05-21 07:53:06.829103969|2030-09-22
    +3|7603|2038381675|7386087924003676160|-21580.07|933815.29|-3828528440938.576660|false|linguistics|quinn ichabod|2026-12-02 16:54:36|2044-04-29 02:35:27.537827490|2086-05-11
    +-69|-13474|-373038706|3907|-18880.83|-939028.09|117666480427.907227|false|undecided|priscilla ovid|2067-02-17 21:38:22||2075-08-06
    +92|2833|-2119724898|7818464507324121088|30292.00|884488.54|871664464870.037109|false|joggying|holly young|2065-11-24 15:01:57|2071-09-02 12:05:40.306146425|2041-05-01
    +21|30075|-1464514590|-8293833565967810560|-36263.23|-1474035.29|-4690640135324.102539|false|quiet hour|nick garcia|2033-09-22 16:37:28|2072-01-29 18:50:14.473666434|1994-05-25
    +119|-12571|-779743333|-7892780594910871552|4572.65|995018.67|-425783093743.490234|true|joggying|ethan white|2061-07-26 18:52:34|2071-07-05 00:55:22.981130585|2029-09-01
    +-83||1920662116|1509|33314.97|-2730529.51|84511300621.094727||wind surfing|tom hernandez|2050-03-09 05:14:35|2078-12-03 09:09:20.319905333|2029-09-01
    +85|-13713||7592440105065308160|42570.35|4483692.57|-2934238933405.062500|false|xylophone band|jessica quirinius|2030-03-23 21:15:46|2045-01-19 02:06:48.899240157|2103-08-27
    +53|17436|-722294882|-7600138468036386816|36893.19|-4542207.94|-1743688633337.790039|false|education|quinn quirinius|2017-05-28 14:36:27|2066-02-21 06:53:29.204553365|2009-09-12
    +3|10727|-1849091666|9064847977742032896|17555.77|-4634541.04|2527955410135.795898|true|nap time|oscar carson|2048-03-24 17:24:21|1970-01-01 00:00:00.846186492|2033-08-07
    +9|20766|-181122344|-8379964450833367040|-35831.96|2510460.11|3065070467778.455078|true|geology|irene davidson|2050-02-22 15:05:52|2027-10-08 09:45:46.728679053|2050-06-14
    +113|-3165|-90029636|7217123582035116032|12022.56|3658068.63|-2382124601444.530762|true|linguistics|wendy laertes|2033-11-09 23:52:55||2070-09-29
    +93|24677|-472303419|-7879864376629567488||-4715822.32|3836587003422.593750|false|linguistics|rachel garcia|2020-12-08 01:26:41|2018-07-13 07:36:49.359511410|2067-01-14
    +9|-3885|-906545548|2878|-38407.73|-1928000.84|-2800032052132.195312|false|zync studies|xavier falkner|2056-02-23 09:06:30|2047-05-31 06:42:08.683378784|2016-11-13
    +-37|23750|-1749841790|2412|46750.89|-2458475.76|4439101915043.935547|false|zync studies|katie thompson|2073-10-11 11:47:19|2026-01-28 22:23:58.98308516|2092-06-27
    +-91|-29218|-1326025787|524|42679.72|-662744.93|310684347775.804688|true|chemistry|calvin garcia|2068-11-17 02:45:29|2059-01-25 08:18:48.682643621|1971-12-01
    +75|21407|44595790|784|-39388.69|-4752379.35|-4679521339883.549805|false|debate|xavier brown|2059-07-30 11:32:09|2029-01-31 01:58:51.459910049|2007-05-08
    +35|4397|-117723745|-7046180371529351168|-22128.92|-1882093.86|604464981663.425781|true|quiet hour|oscar white|2014-11-01 18:19:21|2080-04-14 06:26:44.785796402|2054-06-03
    +-26|-30730|-207899360|471|22281.44|-2302502.73|-1952325542067.043945|true|debate|tom ellison|2041-01-14 15:00:44|2080-03-13 16:08:00.809220690|2063-04-29
    +-81|-7564|1131663263|612|19632.85|3876730.76|-3157294779103.096680|true|study skills|david laertes|2062-12-12 19:12:58|2035-06-20 23:05:28.501460278|
    +-98|21941|-1665164127|8368012468775608320|528.05|3412429.91|-950235709930.815430|true|industrial engineering|victor polk|2043-04-29 14:31:44||2060-04-22
    +75|-716|1784291853|-7547245548870025216|33682.36|-2277813.00|-2651511947479.434570|true|history|yuri falkner|2060-09-06 05:02:04|2015-12-10 05:12:43.991653609|1970-04-26
    +1|8466|-901079162|3841|26403.70|-2787173.70|-4078808993300.313477|false|zync studies|fred ovid|2042-01-27 06:10:24|2065-09-04 01:31:40.42530548|
    +-97|913|-1502924486|8752150411997356032|31964.76||4205989218534.185547|true|opthamology|rachel nixon|2027-03-17 19:37:19|2028-03-14 08:43:52.398149158|2064-03-06
    +-34|15015|1637295757|-8623965248051789824|-13910.96|-1899040.07||false|linguistics||2074-06-10 07:25:52|2064-06-11 04:29:32.55418236|2094-06-14
    +-33|20036|1880017800|7637152193832886272|-13101.15|-4590457.77|-1226073411038.548340||quiet hour|bob nixon|2055-07-25 23:02:17|2072-05-17 10:12:38.513326012|2012-07-07
    +22|-16940|-595769210|9191943992860327936|42065.10|353607.21|-3757504944023.821777|false||bob polk|2049-10-28 12:03:35|2028-12-24 08:45:04.462025798|2099-03-24
    +-81|25454|-123529324|2700|-49851.20|-199346.24|3039471429849.208008|true|american history|luke underhill|2041-05-03 12:44:35|2062-05-13 13:04:05.633293665|2016-04-11
    +111|-6349|1950882901|9180098147855769600|8936.06|4586140.88|-2562228399414.543457||undecided|xavier allen|2047-04-04 14:41:47|2029-10-19 11:17:14.241599813|2076-10-10
    +32|-6564|1928365430|1775|16609.42|3708243.51|-1030172091977.021973|false|chemistry|alice thompson|2057-06-03 18:50:23|2016-01-12 02:37:54.734293795|2010-10-15
    +87|5550|996831203|797|-32213.23|-348044.95|2004890357846.349609|false||quinn quirinius|2025-05-25 18:20:23|2065-11-02 11:48:50.631850931|
    +48|-9943|270090617|-7773957003968675840|39329.34|4135595.75|-243786022935.737305||wind surfing|holly laertes|2030-12-19 19:08:33|2023-04-14 22:34:26.707421905|2043-08-05
    +-115|28301|1317690178|-8660149447361404928|-37823.29|-2814333.34|-3512730030774.058594|false|geology|holly miller|2029-03-26 00:28:37|2039-02-23 12:54:55.42365758|2076-09-26
    +-46|18350|-1531040609|8641221723991433216|23823.69|-427642.08|2819404917896.939453|true|yard duty|rachel davidson|2024-07-29 07:41:06|2025-01-31 13:56:58.481985366|2067-01-10
    +51|-13904|1664736741|392|-33079.41|4087589.11|4145813588397.863281|true|chemistry|jessica king|2035-02-25 00:23:44|2052-01-19 21:04:20.398663046|2079-08-01
    +19|-26613|-971615370||33112.52|4554924.96|-4303832304648.800293|true|chemistry|calvin allen|2073-04-14 09:13:06|2039-12-03 11:31:50.851455297|2009-09-29
    +-89|29333|-1124028213|8489735221193138176|27673.25|-4285255.26|4961373696863.767578|false|linguistics|katie ellison|2072-02-05 07:19:52|2067-07-12 01:46:05.391290184|2018-03-24
    +0|-12203|372099650|7944741547145502720|-6724.11|-3694995.69|-630159357028.894531|true|undecided|luke robinson|2033-06-16 06:37:14|2014-10-03 05:32:19.137455217|2068-06-05
    +111|18910|780859673|6933731240564056064|-44662.14|-1927984.24|-3491883806228.379883|true|biology|katie van buren|2038-08-20 15:28:40|2064-04-16 22:33:47.576062785|1970-06-03
    +57|1280|-1359838019|9083704659251798016|20916.87|-1497681.60|2846930959096.063477|true|biology|quinn miller|2053-01-28 18:41:42|2060-07-25 07:46:20.708072689|2002-05-27
    +122|9177|128430191|-9084940280061485056|-42499.27|3229184.24|-4017461708820.384766|true|values clariffication|ulysses ovid|2034-11-06 04:21:49|2034-07-30 16:22:19.644823674|2023-10-05
    +-78|-10532|-318380015|8222714144797368320|-11740.33|-2082364.30|-2110282626871.757812|false|xylophone band|katie ichabod|2046-04-25 00:51:33||
    +-82|-24320|1550375386|8817665768680906752||-953044.44|-2273400242763.017090|false|geology|sarah carson|2048-08-22 04:37:46||2086-10-16
    +113|-27401|1012230484|1995|40659.62|-4935987.77|-1008868027448.919434|true|debate|sarah polk|2062-01-02 11:36:14||2080-04-14
    +-111|30736||1561|-31482.56|-1580110.12|-1566816988300.167969|true|zync studies|quinn quirinius|2060-03-07 21:51:32|2061-11-09 12:07:37.596342844|2026-11-23
    +-5|-4037|-379643543|2485|-9312.35|3850072.85|2057210782083.452148|false|industrial engineering|david quirinius||2061-11-11 05:38:01.34481671|2021-04-19
    +27|-13623|505902480|1826|-42341.81||2681464429083.355469|true|chemistry|ulysses garcia|2074-12-19 04:00:03|2049-02-17 19:06:31.903289514|2013-01-23
    +|14234|-1026746699|845|35740.11|4705168.38|-4722254262111.391602|false|quiet hour|rachel hernandez|2019-03-25 04:02:54|2080-10-07 11:37:55.957012102|2004-08-13
    +-58|3325|1665724041|8376440110255243264|46586.97|1120362.22|3853942610392.876953|false|history|luke laertes|2029-12-12 07:11:57||2004-04-01
    +-17|22289|712816880|9075404705968840704|-15320.85|-3930645.10|-4500316102425.934570|false|linguistics|alice king|2062-09-20 07:45:00|2021-06-10 09:55:52.478946811|1994-01-27
    +-28|-11740|1566607834|-8379109122834997248|21449.12|-4905990.36|-1952272677926.849609|true|debate|david brown|2018-12-26 15:21:05|2055-07-04 00:03:47.54540038|2058-11-22
    +-102|30420|980732494|-6938706403992854528||4290144.13|3973620252675.828125|false|nap time|holly polk|2013-06-14 16:30:17|2079-04-01 01:20:33.477066908|1973-06-12
    +-27|10473|1805139501|961|540.66|1846904.66||false|study skills|katie thompson|2080-12-15 12:30:07|2018-10-01 01:35:46.827604858|2065-10-01
    +93||-1402821064|1422|-32181.01|1471767.66|3888068884720.255859|false|forestry|luke brown|2055-04-11 22:16:15|2023-10-05 09:42:54.182435030|2001-11-08
    +-72|-31033|-694520014|9149216169284091904|-21438.32|592620.51|-868405318561.400391|false||quinn steinbeck|2038-03-12 13:12:37|2019-08-02 08:13:35.567323306|2015-09-08
    +-83|-21268|962091264|2752|-9102.58|-1920670.21||false|forestry|tom polk|2048-07-06 23:16:50||2103-08-02
    +-91|5014|-1561738723|2255|11659.02|-1238767.70|2257301642704.199219|true|opthamology|fred underhill|2034-05-29 03:12:18|2056-10-14 08:06:18.341166711|1973-02-06
    +-46|24913|906074599|-9080568167841226752|-1633.77|2280305.65|3418787511905.457031|||wendy nixon|2054-11-07 20:23:51|2069-03-08 13:59:21.935346573|2027-06-01
    +-55||-990781312|1046|-17223.20|-2914331.70|-3061715672539.690918|true|american history|victor xylophone|2025-06-24 19:34:32|2029-05-31 09:16:06.592040235|2067-06-22
    +85|14176|1582537271|7926898770090491904|-48480.35|-4206386.08|-3830188857046.366211|false|kindergarten|jessica garcia|2027-03-21 01:29:39|2050-06-01 01:40:27.460760962|2094-01-29
    +5|26915|-158848747|7784489776013295616|25599.75|3264338.39|1398225707031.031250|false|geology|sarah ellison|2081-02-08 13:31:19|2016-12-22 00:48:27.635611363|2019-01-04
    +48|20655|-1345085327|6991316084916879360|5659.91|2971588.36|944624924616.661133|true|philosophy|bob johnson|2042-07-11 18:36:57|2037-12-17 04:00:52.301645045|2015-05-25
    +-60|-23750|747122546|1566|-32617.06|3930932.70|904359634710.800781|true|american history|david thompson|2049-08-15 13:53:19|2049-05-25 04:35:12.968980434|2054-07-15
    +1|19276|1504919241|1671|-35828.00|-1990218.76|1742934975017.433594|true|undecided|calvin miller|2033-10-17 07:51:41|2045-07-16 13:46:26.28793094|2089-07-13
    +76|10382|-607667405|-8543982423727128576|15112.99|-1144078.40|1538907131524.070312|true|nap time|ethan young|2039-03-18 13:48:12|2022-05-16 00:50:53.345008930|2072-03-19
    +20|5947|-66010816|-8832750849949892608|43254.26|-4684380.49||false|debate|rachel davidson|2022-03-02 20:53:13|2029-05-15 20:54:25.943787316|2051-12-13
    +|10083|-1340213051|6963217546192322560|-23734.81||42087741032.439453|false|wind surfing|rachel ovid|2026-01-17 13:10:48|2068-04-09 14:34:20.845374064|2046-04-19
    +25|-11129|514833409|236|14759.04|-4445156.14|-2471323804264.180664|true|chemistry|katie nixon||2062-06-11 00:50:35.200240078|2012-03-25
    +-117|21418|-1106685577|7086206629592252416|-20613.82|-4669723.14|1287664454410.020508||undecided|ulysses ovid|2014-05-27 23:10:23|2075-10-09 13:36:21.138709539|2055-09-11
    +73|-32208|1075444504|9053187076403060736|-20607.86|-663557.15|3604091262736.055664|false|religion|luke robinson|2064-03-29 13:46:25|2028-09-13 12:25:32.422151087|2001-09-18
    +68|-9365|214068706|-8067243114610532352|-40369.97|-372273.18|-4679196929122.006836|true|american history|bob thompson||2016-06-02 13:49:05.762346775|2031-11-23
    +38|29288|-667383951|1751|-40969.80|352106.97|-3590646924755.134277|false|religion|alice white|2021-01-31 08:56:35|2074-02-24 20:30:17.797682689|2054-10-30
    +73|1077|-336625622|2502|34593.52|-3380008.10|4861797358093.511719|true|linguistics|yuri underhill|2031-10-04 05:10:58|2020-09-16 21:07:49.9039871|2000-08-20
    +86|-8927|-1817096156|294|18420.41|-718931.02|-812599933568.947266|false|topology|yuri miller|2030-03-08 13:49:25|2059-08-20 13:04:07.326107637|2026-09-01
    +46|-26138|-371779520|7892281003266408448|8622.19|4294576.25|-2935717014226.568359|true|education|fred xylophone|2025-08-10 03:36:42|2058-08-31 20:49:43.13681707|2017-12-24
    +125|7954||8577096957495025664|-40074.79|-3740147.62|1353404175657.040039|false|education|sarah garcia|2020-04-24 05:38:29|2059-09-29 14:12:16.157602035|
    +|23725|-45460011|-8665764757143658496|-10058.15|3111392.39|4140724331387.878906|false|wind surfing|mike nixon|2022-12-19 01:07:38|2077-04-10 05:41:52.802703873|2039-07-14
    +95|-25151|-1770250407|2855|-8224.46|-1232102.28|1061929942085.519531|true|education|victor davidson|2024-08-05 07:04:05|2076-04-22 18:20:42.949891304|2082-07-08
    +-92|22075|-170643477|2811|-6333.87|-2996672.74|-4596379165591.137695|false|education|oscar johnson|2075-03-08 07:27:49|2031-01-22 04:35:56.338245755|2001-02-13
    +-107|15530|1028092807|8785153741735616512|23063.22|4917736.71|-1387735872479.432129|false|study skills|ulysses quirinius|2051-06-21 00:48:20|2064-05-26 10:29:04.116652584|2001-03-24
    +-35|-4509||1726|-36045.69|-957009.39|-1436845214707.389648|true|industrial engineering|katie ichabod|2061-12-03 22:11:19|2037-09-03 21:28:18.120030510|1976-03-25
    +10|8243|1430614653|7186401810812059648|21430.58|-4117132.25|4273903933165.144531||values clariffication|david brown|2080-09-06 01:51:56|2067-02-25 00:06:03.464200151|2008-03-31
    +-100|-18358|390124976|-7603569103205916672|9320.42|820509.76|-2378674120573.869141|true|philosophy|gabriella laertes|2054-02-15 22:48:03|2023-07-16 09:03:23.416142255|1987-01-17
    +-34|-12896|-396852483|4018|14230.43|1948755.04|996493613976.418945|true|american history|jessica nixon|2035-11-08 16:07:43|2042-02-21 13:24:08.768185439|2024-08-27
    +-79|-3883|-519978947|3566|-25444.08|-4338469.48|-2225952289175.303711|true|biology|bob van buren||2053-09-12 14:38:23.627646057|2061-08-19
    +38|11067|257821327|2725||2123890.76|-4311579608982.931641|true|quiet hour|bob ellison|2074-07-08 16:12:25|2028-02-16 07:26:07.314316265|2046-10-04
    +-46|9447|-1921909135|1234|43332.86|-1313189.64|3483740651742.017578||debate|ethan hernandez|2053-12-13 02:32:42|2024-05-02 15:41:49.244300741|2012-02-14
    +|-31217|-1880877824|346||-4470321.21|-2089255269314.307617|true|zync studies|tom quirinius|2032-11-07 23:12:11|2057-06-05 07:40:56.143268958|1986-12-22
    +||866084887|7961515985722605568|-13131.71||-2394101676301.012695|false|wind surfing|david polk|2052-08-30 08:01:09||2052-02-02
    +21|32611|482977302|7274777328897802240|-6306.38|1252331.81|-533108289688.186523|false|study skills|jessica robinson|2061-05-19 17:30:01|2079-02-13 03:05:01.151556875|2064-10-20
    +-80||581259902|-6933565857643814912|12765.75|-3696105.78||false|wind surfing|yuri ellison|2032-01-12 17:09:47|2026-12-04 01:38:04.606186772|2046-12-26
    +62|-776|-409404534|-8330233444291084288|-13235.88|-4942351.80|1478986789735.259766|false|wind surfing|luke nixon||2015-09-27 04:55:14.418545051|2079-08-04
    +-15|7988|1969650228|34|49433.36|-1454704.21|3304426642736.508789|false|debate|bob white|2032-05-25 11:24:01|2050-08-06 03:07:56.829882190|2011-02-25
    +-100|-19374|-337073639|7080269176324218880|9636.84|4955437.13|-1605703810903.698242|true|joggying|priscilla nixon|2016-06-18 01:36:41|2076-06-24 17:32:46.629540610|2051-08-11
    +31|11050|-1862095575|2941||4773173.81|-4916009895587.691406|false||tom ichabod|2026-02-05 06:06:27|2033-11-01 09:06:22.861384004|2064-04-06
    +42|29954|-297664578|9117063974299148288|-36654.25|1809785.84|-2990108911038.724609|true|linguistics|quinn zipper|2063-01-04 13:11:49|2032-04-05 17:24:19.790913068|2082-02-07
    +36|-1307|-603273425|-6917607783359897600|20495.55|-717022.00|401293377521.448242|false|xylophone band|ethan steinbeck|2043-08-16 10:35:12|2074-08-29 01:50:39.79906002|
    +-47|20307|-1409508377|-8566940231897874432|-23112.40||-4906953690985.069336|false|american history|katie laertes|2018-04-24 10:05:45||2070-04-05
    +-27|16664|-1817938378|-8710298418608619520|-19481.20|2004949.48|-2008363741307.021973|false|philosophy|sarah laertes|2038-09-29 23:53:07|1970-01-01 00:00:00.626944877|2041-07-12
    +-31|20391||1520|39315.23|4922532.64|-1940382804363.878418|false|industrial engineering|priscilla brown|2045-10-19 14:00:02|2047-12-07 08:51:00.336787227|1988-07-12
    +113|-21146|-800799595|3728|-6051.92||4288442203475.107422|false|history|luke ovid|2042-04-07 13:12:41|2017-11-07 00:47:02.498508838|1980-07-10
    +127|-26526|-938112972|-8835408234247168000|-5042.40|-639514.18|2253781196684.378906|false|topology|rachel polk|2059-06-15 22:27:11|2078-04-15 00:06:23.865331718|2002-01-27
    +47|3374|527598540|7705445437881278464|5750.75|265215.83|-3997464183110.714844|false|topology|luke ovid||2055-04-30 05:00:45.288293151|2038-08-04
    +-57|7826|987734049|6926925215281774592|4092.06|-4043613.94|1825916166900.114258|false|wind surfing|alice hernandez|2066-02-26 20:09:56||1994-01-31
    +30|-4159|-1054609414|835|-24688.28|-3625707.02|-1643333177209.917969|false|biology||||2097-09-12
    +23|1326|-199587670||23788.91|-602696.25|3307871238283.475586|false||priscilla miller|2017-03-07 08:16:25|2052-04-04 23:29:00.786514521|2091-03-28
    +59|3270|1434588588|3232|1480.61|743880.56|2593124571056.222656|true|wind surfing|nick allen|2056-12-31 22:55:26|2076-06-29 19:41:36.482632437|2051-05-15
    +66|3245|-300717684|-7840338174858199040|40071.10|-905100.40|-4829089800174.344727|false|wind surfing|sarah ovid|2056-03-09 19:09:50|1970-01-01 00:00:00.59977853|2047-10-04
    +85|6074|-540401598|7748799008146366464|20761.91|-4916297.37|2180818539557.698242||joggying|david brown|2024-11-21 14:50:04|2028-01-24 01:00:27.143579402|1970-07-16
    +38|-7948|1987336880|7410096605330227200|18427.96|1858816.32|2290676735205.039062|false|topology|fred ichabod|2052-07-29 14:33:52||2090-05-17
    +-127|11451|316438994|188|6220.87|2395035.62|1762390956829.769531|false|american history|sarah white|2035-12-04 05:59:54|2037-10-21 10:24:49.395474612|1976-11-08
    +-49|19041|-1464762852||-9900.51||4912867697965.523438|false|biology|bob garcia|2080-03-12 06:31:47|2079-09-20 05:14:58.169640521|2008-01-13
    +-104|5191|595836061|-7709958788604936192|-17146.91|-3981389.85|1420240216579.884766|true|nap time|priscilla johnson|2018-10-25 09:42:50|2027-12-11 09:57:50.980641227|2019-11-03
    +115|-26832|-764412063|-6920172215209426944|17394.03|561210.22|997882374014.622070|false|linguistics|victor falkner|2018-12-21 14:56:23|2074-08-29 04:26:14.412387056|2100-03-23
    +-84|-16327|407098216|-9109392978217484288|24409.31|4875980.88|2156284924808.034180|true|undecided|priscilla garcia|2049-08-03 07:20:14|2072-03-18 16:05:07.216021739|2020-01-11
    +56|21814|773730574|3608|19617.72|2612031.03|472615987880.900391|true|values clariffication|holly polk|2079-07-31 02:12:53|2033-04-30 22:04:27.178988903|2065-03-04
    +109|-5097||-8214462866994339840|14926.06|4194003.54|-3157298893601.385742|false||ulysses falkner|2051-01-12 23:18:22|2056-07-05 00:35:53.936899939|2083-09-24
    +31|6900|-604362582|2306|25232.04|-301550.41|289949139351.364258|true|zync studies|gabriella ichabod|2041-03-14 14:44:09|2068-11-06 15:41:10.293161863|2098-08-25
    +-122|30119|-2065287410|-7759238919361888256|-48628.97|4508399.10|-1946145505911.474121|true|topology|rachel quirinius|2042-12-15 05:57:04|2057-11-03 01:20:57.262442758|2077-07-27
    +-81|32567|-536315467|-8922409715403112448|-49532.02|-2541992.13|-366989953592.735352|true|forestry|luke allen|2059-07-26 01:05:11||2075-01-17
    +58|-10247|-186600427|3664|-20024.15|2499207.86|3164512989036.332031|false|history|sarah thompson|2036-09-10 12:17:11|2064-10-23 01:33:56.949763617|2085-10-20
    +87|-22431|-1625800024|-9203942396257984512|10700.82|-4871285.36|1078033806441.389648|true|zync studies|fred falkner|2079-08-03 03:53:40|2077-02-22 11:17:37.324498592|2090-11-24
    +89|24782|1914993018|8116738401948377088|28477.87|4428379.36|-3581726665433.642090|true|linguistics|yuri laertes|2042-05-04 19:59:28|2015-11-03 09:21:31.868284245|
    +-19|-25973|-1900369503|1791|38658.65|3263702.68|-971581426672.542969|false|history|gabriella thompson|2019-12-18 15:34:18|2059-11-10 14:30:30.756815026|2094-09-03
    +112|-5635|-4393552|-7419068456205385728|-16898.67|-3147119.24|-4071056405979.831543|false|history|rachel ichabod|2020-05-25 05:03:07||2041-02-27
    +6|-5946|1366402722|8795069490394882048|-5903.91|-302873.33|-3374663867091.989258|false|quiet hour|gabriella robinson|2053-11-12 16:58:09||2071-08-09
    +-115|325|692666133|3043|-22120.22|3867286.85|3593749673935.349609|true|yard duty|mike miller|2030-09-24 22:40:43|2059-05-19 21:39:40.717511206|2054-10-11
    +46|-24248|-1871446009|3174|24920.44|-2650736.01|3646290606200.395508|true||david johnson|2021-12-02 02:13:49|2027-06-25 15:06:55.689498978|2092-05-03
    +-92|28558|-1699044525|7625728883085025280|-26098.47|-201231.45|-879898512524.555176|false|philosophy|nick zipper|2039-09-05 02:14:13|2067-05-11 07:05:39.707943033|2082-09-07
    +22|8786|-1196808950|-8585134536083660800|36606.56|-1258722.35|3594939606490.008789||philosophy|irene polk|2033-07-09 03:38:52|2070-09-24 03:28:58.997336852|2069-12-08
    +6|-30638|-370901197|8558000156325707776|-9840.93|-4341927.96|1094936081745.956055|false|linguistics|katie laertes|2078-08-17 11:00:57|2047-01-10 07:06:46.173146425|2028-12-05
    +48||1787826883|-8572949572756774912|-7447.88|-4657467.74|-1120139371327.120117|false|kindergarten|sarah allen|2045-11-09 08:09:15|2041-12-23 20:40:16.590539947|1996-09-10
    +-2||1045719941|661|48481.57|-199335.41|1697710053431.365234|false|wind surfing|oscar allen|2077-10-09 07:08:17|2044-08-23 18:37:23.765446063|2039-05-20
    +-18|-2636|1852725744|2393||-327510.29||true|opthamology|david garcia|2071-07-30 15:03:51|2049-10-01 11:47:21.417323976|2022-09-06
    +108|-8578|1352739140|-7800879252150779904|27366.30||-180060584501.582031|false|history|luke miller|2067-12-30 02:21:40|2029-03-05 23:41:09.158607395|2027-02-24
    +70|-30163|2044130430|7534549597202194432|-3684.66|3361124.51|-544906712728.980469|false|debate|luke brown|2015-07-17 03:26:40|2054-07-10 04:48:29.103251644|1981-06-25
    +-72||583458404|-7642381493746483200|-41758.86|-432625.76|739045453673.981445|false|xylophone band|fred falkner|2075-02-09 15:25:34|2024-10-11 18:27:50.785894213|2038-05-15
    +-31|14335|-2024003241|-7330413050756235264|-20201.82|672555.54|1469464763006.626953|false|topology|||2044-04-09 14:18:23.656514547|2038-02-20
    +-44|31242|605946758|7596563216912211968|30099.30|2097061.15|3078551005937.946289|false|chemistry|calvin ellison|2040-08-01 15:36:49|2034-04-12 16:31:19.919073203|1981-01-10
    +-115|-5240|-1128317466|3307|-17763.36|-3640432.08|-3634683485380.314453|true|chemistry|ethan young|2014-07-01 14:59:22|2065-10-29 07:42:38.763916292|2102-08-12
    +-60|4725|-373541958|2971|-21091.82|3177096.44|2070825437177.284180|false|values clariffication|priscilla quirinius|2030-01-02 10:09:12|2017-02-05 15:11:27.916015499|2098-04-30
    +-35|-3427|1533817551|2285|22842.27|434947.31|-233465238088.080078|true|biology|david underhill|2069-11-27 12:11:44|2038-10-25 03:53:58.936213000|2018-10-02
    +23|-32383|1293876597|1880|15273.59|1943186.18|1318023503084.724609|true|quiet hour|mike brown|2036-06-15 05:33:48|2035-04-17 20:50:04.917479654|2086-09-08
    +15|5972|947846543|4088|-24858.15|-1682574.63|2790323958356.459961|false|religion|quinn garcia|2039-02-08 09:45:49|2051-06-08 01:29:12.240654179|2020-09-22
    +18|-10277|1004241194|743|37468.44|90556.54|-1721911019841.009277|false|study skills|calvin hernandez|2017-11-06 20:56:23|2066-11-30 20:25:38.323872312|2087-05-31
    +96|-28730|-670925379|-8317591428117274624|45783.32|-4491871.75|1347547267070.333008|true|religion|ethan ovid|2039-04-05 21:26:27|2047-02-15 03:00:04.876322903|2016-12-02
    +49|22511|1301997393|8854715632851345408|-7866.49|-4806269.72|4690235057502.966797|false|geology|victor van buren|2077-11-06 21:58:21|2026-07-12 18:24:40.599693692|2023-01-30
    +116|21606||7768984605670604800|-37668.86|-4451507.54|1896615693548.895508|true|chemistry|victor steinbeck|2049-10-18 09:39:20|2017-04-25 06:11:37.174630171|2045-09-29
    +102|-26461|2114363167|2900|42096.38|273171.55|-2395416839539.450195|false|industrial engineering|calvin falkner|2038-01-28 15:37:14|2021-10-28 12:53:49.438479282|2012-09-25
    +-35||-1614194712|7062605127422894080|10143.00|3278700.42|-8877240475.761719|true|debate|alice laertes|2047-08-19 00:56:41|2041-07-30 06:11:21.601946450|2060-10-26
    +30|-23672|-1983567458|7394967727502467072|42769.25|245529.05||true|yard duty|nick laertes|2052-08-02 11:29:42|2048-02-09 16:35:20.385399131|2092-01-30
    +60|22378||1781|39850.06|-4048951.22|3015653383409.677734|true|yard duty|gabriella ovid|2054-01-29 18:44:50|2063-07-31 19:03:05.155926850|2006-12-12
    +38|-12193|-1770229099|7238339720750948352|10757.18||2824970709025.791016|false|philosophy|alice miller||2058-07-11 09:32:37.491928305|2001-09-05
    +|-2287|92777932|1638|-25700.79|-181051.84|4692362804086.371094|true|american history|ulysses davidson|2037-12-15 16:39:08|2075-09-17 23:16:13.795444412|2079-12-10
    +93|15158|1475025489||8969.07|-4073690.87|909614012102.934570|true|geology|alice davidson|2074-01-01 16:32:52|2019-06-28 19:19:52.904542868|2048-06-04
    +|6899|633813435|-8522878384019169280|-8564.75|4996750.69|755521930470.130859|false|forestry|oscar king|2078-04-19 11:14:16|2029-01-24 18:55:58.605673322|
    +78|-9398|51376784|-8051587217208967168|42110.77|4237872.52|-480292543988.199219|true|religion|fred young|2043-08-31 12:19:13|2027-11-28 17:42:04.124425023|2043-05-27
    +-98|22678|-765102534|-7425160895830573056|-36404.91|-232802.04|70171166399.717773|false|mathematics|oscar king|2061-03-29 17:11:56|2025-01-04 05:18:13.67002718|2070-03-30
    +115|12263|-1669848306|7344029858387820544|45009.74|-4145959.26|-1287206623226.708984|false|study skills|holly thompson|2030-07-06 18:59:28||2068-03-31
    +-98|-18295|-1391183008|-8013397854633648128|-12751.03|4087534.61|1251848312304.524414|true|values clariffication|ulysses brown|2042-11-01 14:59:40|2050-08-30 06:33:17.144659188|2100-07-17
    +59|297|-1706867123|8808467247666241536|-23178.82|1057885.16|4299677631158.154297|false|industrial engineering|calvin allen|2017-05-25 20:58:54||2071-03-31
    +-68|-26481|-726879427|-8768744394742235136|-46383.26|1261211.13|-2782916269764.896973|true|xylophone band|calvin young|2030-05-23 20:23:28|2051-07-05 23:08:45.51977303|2013-03-12
    +-6|31316|-1011125931|9185458640237641728|-17508.25|-658586.36|1020511437056.228516|false|industrial engineering|zach ichabod||2061-07-05 08:12:13.181218526|2006-12-06
    +-51|-20182|-892839693|-7686220526274502656|45425.18|1655396.08|3720881118135.359375|false|yard duty|wendy davidson|2065-08-25 22:34:57|2077-06-29 07:51:23.99926741|2003-09-22
    +93|15815|-626484313|-8203075743525806080|1895.94||977607524337.074219|false|nap time|ethan underhill|2066-01-12 20:36:33|2016-07-24 00:19:40.599709326|2040-02-16
    +122|-22390|511836073|3462|-13499.91|-2876022.64|1328013667669.125977|true|opthamology|calvin ellison|2038-09-26 01:22:16|2050-10-02 05:27:49.739848630|1997-07-24
    +31|10544|2146312499|6964585306125008896|-1730.59|1232330.61|4495354594042.523438|true|nap time|calvin allen|2051-08-10 11:03:28||2099-02-09
    +-125||-940504641|3418|-42213.02|1998604.86|-2596755613328.838379|true|chemistry|victor robinson|2069-06-09 00:13:59|2049-07-11 09:28:46.795925700|2066-06-17
    +-55|21440|-606214770|3366|-7764.69|150515.54|3713473719274.773438|false|philosophy|holly king|2065-10-25 16:36:46|2058-03-17 19:37:18.893032333|1998-04-11
    +|-11123|-1078579367|-7867219225874571264|3544.10|-4440011.92|-1597354768432.488281|true|education|katie nixon|2021-12-20 01:53:24|2016-04-25 20:51:22.134986617|2009-08-13
    +14|-12517|-1269216718|8367680396909404160|41509.01|-1881292.71|-2614008444640.525391|true|topology|wendy nixon|2031-06-01 22:05:01|2019-05-30 05:53:53.829470930|1986-10-09
    +-78|-7027|-664856187|7524958388842078208|-49058.42|2299272.82|1193868149375.028320|false|kindergarten|alice xylophone||2038-05-26 15:00:22.242283005|2072-09-03
    +-69|4718|-47662800|2897|32798.27|3454639.78|1810203200294.040039|false|philosophy|zach polk|2069-08-29 08:29:09|2014-11-20 05:08:33.941609402|2082-06-01
    +-72|-15957|-630900418|8391785334471589888|-49753.88||-3594602463335.583008|true|geology|oscar steinbeck|2034-06-08 17:12:48|2048-09-23 09:12:15.792761284|2039-09-20
    +-57|15379|-674478103|-8581979259158929408|24804.72|516411.04|-2087053480555.418457|true|debate|ethan van buren|2039-03-15 22:10:39|2059-02-19 09:59:56.329119790|1980-04-11
    +116|-4799|1485934602|587|-36250.77||-3139538476660.561523|false|zync studies|tom garcia|2070-11-21 14:24:06|2077-06-02 22:53:29.588878692|2021-02-13
    +5|20183|-2081501748|130|16581.21|2049671.63|162654497074.683594||opthamology|priscilla falkner|2022-12-17 04:17:53|2050-08-15 19:53:57.340023498|1985-05-04
    +25|-17429|-300429552|1030|48749.30|891980.69|-51603607081.567383|true|yard duty|calvin johnson|2017-11-20 04:16:35|2055-03-24 21:11:15.57401629|2053-12-19
    +45|-31764|89366322|8362046808797306880|-28518.05|2976251.39|2342010012983.273438|false|philosophy|wendy underhill|2015-07-02 23:30:23|2076-07-01 17:36:04.583971136|1999-02-20
    +124|3714|-664111469|3691|-30852.26|2707139.90|-4940400326044.844727|true|debate|katie king|2029-04-17 21:35:26||
    +-97|4749|859140926|7454632396542074880|46493.77|4790547.07||false|values clariffication|jessica robinson|2072-02-26 01:04:57|2056-06-21 06:38:40.564403087|2042-05-03
    +104|-7467|-2111312205|7125231541858205696|9390.22|-4975282.68|-2004428657765.586914|false|nap time|katie nixon|2051-03-30 14:45:15|2039-09-13 12:20:18.915778346|2105-04-10
    +51|-31881|1933545427|2580|-38988.06|-4829221.83|3300559907076.631836|false|forestry|victor allen|2064-06-06 09:21:23|2070-06-29 22:13:10.207059753|2066-02-22
    +63|17680|-1164833898|2512|18258.24|-1641179.75|752962351021.224609|false|study skills|rachel white|2056-09-13 22:53:52|2044-06-06 09:44:23.311696677|1978-12-09
    +-50|29999|-290558484|7061498706968428544|1510.13|-4232212.06|-4574735317597.173828|true|debate|ulysses allen|2037-07-06 08:01:26|2050-06-04 17:30:11.711624931|2005-07-17
    +45|-21005|2100839074|-7255686273677328384|34455.77|-1253884.38|4472794135666.830078|true|philosophy|calvin ovid|2070-07-26 14:31:09|2036-04-08 23:28:54.417802561|2044-10-24
    +-15|14982|-1079231269|9048002942653710336|-22879.28|4867276.99|-3098102239046.767090|true|american history|victor carson|2059-03-26 12:10:56|2014-08-29 13:16:50.336506060|2012-02-05
    +52|19003|1830870769|8868529429494071296|36316.44|2540170.52|76874893588.936523||kindergarten|ulysses robinson|2053-02-19 15:45:55|2062-12-03 14:50:03.474294016|1984-03-16
    +103||-1701502632|8815398225009967104|16082.15|3286175.37|3474939567092.790039|false|religion|mike xylophone|2036-09-01 16:20:32|2049-09-25 10:34:17.329238191|2015-12-01
    +54|29171|-283378057|7128222874437238784|5784.30|-2487846.11|1273026486246.777344|true|study skills|ulysses xylophone|2065-03-26 18:47:07|2046-10-10 08:53:02.163030900|2065-06-02
    +-29|-22000|826143442|8371939471056470016|-9954.17|1165753.75|-3327839964882.955078|false||zach zipper|2025-04-26 13:39:51|2033-12-28 12:16:51.968462869|1997-05-08
    +45|-6045|-1562552002|-8335810316927213568|18734.01|-1945504.67|-3344796345914.708984|true|undecided|katie brown|2057-08-05 00:41:39|2022-11-18 17:27:46.282162052|2040-01-17
    +-37|-13426|-876122064|-7144791190333546496|-27443.20|3276942.49||false|zync studies|sarah ellison|2027-09-21 21:34:07|2028-07-26 18:55:38.467639955|2046-09-01
    +84|-13038|1107502179||26203.53|1826616.72|52385263076.928711|||oscar brown|2062-03-24 05:55:44|2080-06-08 12:00:45.876270498|2068-05-03
    +-37|16654|1652349607||23892.92|1116670.95|2593325712512.423828|true|chemistry|victor young|2079-05-05 06:48:25|2052-09-05 13:46:09.928597565|2077-10-05
    +5|30730|-841268868|-7572962089372991488|-40151.14|-600165.15|4768177899583.748047|true|geology|ulysses ellison|2024-10-07 05:46:34|2076-04-07 07:58:46.169024390|1972-09-03
    +21|-20657|1503176016|8850055384477401088|-36132.96|3738524.26|2378765399686.458008|false|geology|david nixon|2029-11-13 14:57:22|2018-08-17 10:16:14.518949916|1992-03-07
    +107|-15779|1620529246|2626|10150.78|3475802.90|-2294284421075.900391|false|study skills|katie underhill|2037-09-11 08:16:17|2058-04-21 23:50:14.231746612|1999-03-02
    +-27|-25112|2069258195|3599|3093.83|3254136.59|523934597183.667969|false|opthamology|nick allen|2080-02-16 07:15:57|2060-05-16 16:45:20.894463536|
    +-118|-16813|-361944328|213|-40953.43||-4751644152447.696289|false|undecided|david white|2070-12-23 10:54:21|2048-10-22 12:20:49.521887540|1979-08-27
    +17|28606|277582670|2232|45641.30|128509.04|4503808618217.277344|true|education|tom underhill|2080-09-02 15:00:49|2037-02-08 08:15:49.8202484|2080-09-01
    +102|-18601|283618733|-8297230235506343936|-1052.55|-2765868.79|4944203659075.371094|true|geology|quinn ovid|2075-12-13 06:47:46|2023-03-25 01:12:26.114510579|2001-02-02
    +-110|29515|-395499919|3430|-39432.90|3255575.14|-1079961002014.897461|true|chemistry|wendy zipper|2032-05-15 10:18:11|2062-08-26 11:43:13.962436079|1985-01-02
    +|-9375|1107258026|391|39049.41|2111980.57|-1263586613411.327148|true|education|quinn steinbeck||2028-11-03 23:51:03.652949047|2089-06-26
    +96|28011|830944953|-7395343938785738752|-17062.37|-4047697.78|2447642338433.482422|false|chemistry|irene davidson|2070-06-10 17:13:05|2049-03-17 04:12:33.703429425|1977-08-25
    +74|-14836||9038087402564657152|46621.33|1011931.42|2571176224975.069336|false|american history|oscar nixon|2056-02-11 14:02:01|2061-02-10 07:18:47.459454314|2066-11-29
    +-42|-15695|-1052493316|-9013952631912325120|24814.36|225913.19|1857357798507.415039|true|kindergarten|mike van buren|2032-07-31 17:27:25|2020-07-02 01:16:38.343352835|2081-07-07
    +-80|-11081|440393309|3446|-27153.14|-3115694.15|3504105610850.736328|false|geology|sarah ovid|2080-11-21 06:41:56|2022-09-23 02:34:19.579830812|
    +5|-22582|-397683105|-8703026916864802816|-21676.69|2972179.30|-3159033035088.328125|false|||2016-03-08 14:54:40|2039-10-18 00:49:44.471616221|2045-10-19
    +92|-27715|589546540|-7833618000492109824|14121.48|-2112484.76|-2346132497060.402344|true|geology|david hernandez|2070-12-19 18:40:20|2030-10-05 20:00:42.51205588|2019-03-26
    +60|-23853|-1430903652|1541|-22380.78|2905459.89|-4830217659481.838867|false|xylophone band|bob zipper|2044-11-07 17:03:34|2040-06-07 07:46:11.981745178|2022-05-14
    +-2|-373|435426302|8759184090543857664|36669.53|-4226813.46|-2014561647846.981934|false||ulysses hernandez|2019-01-12 08:00:20|2038-11-27 08:44:14.384170737|2015-01-09
    +121|22333|658636280|-7042183597114081280||1390895.25|3030971793285.637695|false|wind surfing|alice white|2069-01-21 19:40:51|2030-03-05 11:21:30.696290529|2045-07-03
    +25|-10312|1759741857|-7147490721376591872|11100.55|-1549045.77|4731512443034.197266|true|joggying|nick steinbeck|2020-10-20 08:00:11|2061-08-30 15:56:42.855551352|1998-03-27
    +68|25170|1911834442|3725|-46855.76|1255180.75|-1442847310094.883301|false|study skills|priscilla white|2031-05-04 10:11:19|2037-03-01 08:47:26.266061470|1976-03-08
    +85|22852|-1138530007|7961909238130270208|-6093.71|3270007.69|-2674772136505.545410|false|education||2030-05-16 10:48:35|2068-01-21 12:11:12.864391307|2077-01-10
    +116|7258|-966979668|-8930307926221807616|16695.39|-3896225.81|||education|alice falkner|2026-04-02 23:02:34|2061-07-08 03:11:42.545676274|2004-05-21
    +-127|9863|1516149502|2719|-41428.66|4157540.50|638265920153.422852|true|yard duty|david ichabod|2017-04-04 18:50:25|2050-03-30 05:09:44.583713357|1976-02-24
    +-68||-1230459100|-6988970700649168896|-32345.84|-4878754.52|2626801691463.143555|false|forestry|mike laertes||2015-12-05 16:38:41.173038216|2046-11-10
    +-86|-22421|-345542922|-7155539549555105792|42860.19|-4587464.51|2546447982029.782227|true|biology|tom steinbeck|2076-09-06 07:07:12|2057-12-31 11:27:12.154351101|
    +123|-1433|-1656822229|3625||1784691.26|-1490403618753.184570|true||priscilla ichabod|2023-04-14 13:50:19|2022-08-17 04:36:05.763580269|2019-11-15
    +67|-17254|129675822|8113585123802529792||1292322.18|-2056228636604.906738|false|religion|alice ichabod|2075-02-23 01:44:22|2019-09-04 19:27:52.964797916|2101-04-17
    +116|18354|2066707767|9207927479837319168|11869.69|782908.70|-3561486225626.175781|true|kindergarten|zach thompson|2036-01-14 22:33:10|2078-12-19 21:06:17.223542373|1981-03-03
    +2|-6487|-2011708220|-8387347109404286976|3033.19|1933471.30|4046390706154.064453|true|biology|katie zipper|2024-01-09 10:21:56|2048-02-11 17:28:42.54177107|2069-08-17
    +-53|-16052|-768305191||17946.80|303090.34|-364312091649.842773|false|linguistics|rachel johnson|2027-04-27 06:12:38|2052-06-15 15:08:20.185912632|2073-07-08
    +-54|-31404|1540680149|8896237972875370496|2155.10|-4075392.96|4275511770663.023438|true|kindergarten|mike white|2053-02-17 10:06:35|2033-01-10 10:54:27.769712510|2019-02-12
    +73|-29468|564349193|8372408423196270592|8882.78|2172977.04|335964103970.418945|true|debate|tom laertes|2070-07-27 23:21:50|2060-05-03 18:17:59.362453382|2082-10-06
    +28|-16425|932774185|922|36886.25|4970929.84|-2211537110226.693359|true|yard duty|bob king|2042-10-20 10:28:56|2037-10-28 21:57:54.89341833|2079-05-16
    +-108|-5118|1281159709|7255302164215013376|43517.90|-370366.94||false|chemistry|priscilla davidson|2074-11-03 03:30:08|2027-10-18 04:36:01.205338700|1994-01-16
    +-31|-16978|318631333|-8585966098173870080|18570.09|898302.80||false|mathematics|ulysses davidson|2062-03-03 06:35:19||2095-12-10
    +-111|-20112|1847210729|8424515140664360960|-13301.82|-3521114.58|-1467777131906.373535|true|quiet hour|sarah polk|2035-03-23 23:34:06|2068-12-06 16:35:56.147946401|2036-05-11
    +-118|3486|-76654979|-6997233584896229376|-43892.40|1141130.14|3496895382259.453125|true|mathematics|zach van buren|2078-06-17 15:41:41|2017-02-03 01:26:03.765359003|2079-06-11
    +-1|18900|-2137168636|8087737899452432384|24766.64|-1466134.14||false|industrial engineering|katie garcia|2072-04-07 08:27:00|2046-08-31 17:44:23.281747989|2038-04-03
    +-60|-4501|557053197|1493|-28572.29|-3264979.44||true|topology||2069-10-07 09:02:18|2031-11-14 01:37:26.277625746|2093-10-03
    +71|27960|-859535015|8779711700787298304|43414.93|-4284284.11|23723101115.607422||forestry|bob garcia|2065-07-01 07:28:35|2040-10-27 09:40:45.325661811|2078-10-25
    +74|-17056|672266669|2533|14993.12|4181055.99|2330393617025.130859|false|values clariffication|jessica underhill|||2066-04-21
    +105|8328|-44559184||34668.32|-3358597.04|2107086403699.476562|false|industrial engineering|priscilla garcia|2073-01-23 05:17:11|2041-01-24 08:27:17.399682734|1982-09-04
    +35|13020|-1491722659|8017403886247927808|-27501.34|-3084795.60|1361019533518.934570|true|biology|ethan laertes|2049-12-25 02:26:54|2057-08-09 18:37:09.476222463|2091-03-14
    +|-18151|-1140071443|1282|32571.05|-218555.28|2241361269159.725586||kindergarten|mike laertes|2038-07-20 14:27:31|2076-01-14 17:32:40.202975480|2076-03-16
    +-55||-1960344717|2177|-44515.88|-75942.87|873619012141.020508|false|opthamology|holly steinbeck|2037-12-11 11:02:50|2068-04-24 14:56:03.866851484|1973-07-26
    +-74|-12876||-8632237187473088512|27660.85|-3161619.93|3454616098419.087891|false|kindergarten|nick laertes|2054-07-06 18:44:09|2049-03-20 15:41:29.13229610|2043-01-06
    +-121|-11110|2022944702|8109381965028548608|4900.65|-1214641.94|-782319272106.413086|true|biology||2077-04-19 06:19:04|2080-04-29 19:51:25.500331032|2029-03-19
    +29|67|590719541|1157|36502.48|-4871524.01|-2077483805740.065918|false|wind surfing||2034-08-25 19:02:03|2037-08-20 21:02:49.427519908|1992-09-10
    +-45|-24911|1870464222|7378993334503694336|-16461.48|2813386.84|-2450662360123.097168|false|quiet hour||2024-08-11 09:42:40||2059-08-24
    +-72|2074|1013517056||-42455.06|-142043.75|3719431660678.935547|false||xavier ovid|2023-08-30 07:09:18|2018-09-20 23:47:19.977608647|2007-01-02
    +-110||-1946023520|2560|-46654.69|407244.54|-622449463768.788086|false|industrial engineering|irene brown|2049-12-25 17:32:04|2044-10-31 04:37:40.524530302|2017-04-21
    +-71|2335|1003667927|4037|-40079.96|374952.12|4337472356507.205078|false|topology|mike polk|2018-06-24 18:36:35|2064-11-10 09:43:52.16046799|2028-12-03
    +-54|-4364|1775355987|-8562524688907485184|-10552.10|-494146.63|2527866513015.055664|true|nap time|david ovid|2036-03-05 02:43:58|2033-02-15 05:01:02.667963573|2061-02-21
    +-12|-24847|1471913583|2325|26530.25|-11708.56|2137890018516.585938|false|zync studies|rachel robinson|2041-10-21 00:45:05|2031-10-12 16:59:24.57008285|2099-02-12
    +48|-4923|2051470532|6962726713896484864|-31562.99|-3107233.90|1876626916355.661133|true|american history|rachel zipper|2047-01-05 01:41:41|2067-01-14 02:21:16.602793258|2104-10-13
    +-50|-31709|-1379039356|8120593157178228736|20553.47|-2588951.95|3745237629497.139648|true|geology|yuri white|2072-09-13 15:30:55|2040-11-14 23:29:32.319041291|
    +87|26324|-1709117770|6924820982050758656|10187.76|-787959.05|-455009287847.438477|true|opthamology|calvin miller|2037-08-01 12:37:21|2059-09-14 10:15:36.572649743|2083-11-10
    +-32|20746|1592153312|-7366430883634929664|-48828.20|3542628.51|-4174317808621.062012|true|religion|ulysses young|2022-07-28 16:59:23|2055-09-14 11:55:11.844515490|2032-03-31
    +|-23284|-2071851852|-7209060152494817280|34473.73|698236.93||false|debate|ethan ichabod||2039-04-13 07:12:39.959754595|2051-04-11
    +-79||488559595|-8689606130068611072|46873.75|-4585927.20|1922555768632.854492|true|history|ethan hernandez|2016-07-20 02:40:21|2014-04-06 14:33:25.947936182|2006-02-12
    +-124||297577612|3190|26589.02|2183845.19|-1193317944887.937012|true|geology|bob ovid|2014-05-18 11:54:56|2059-07-31 15:56:05.368353709|2063-06-28
    +43|1762|-2119539915|3725|-45749.50|3070034.88|-3994813234986.907227|false|industrial engineering|holly carson|2039-11-03 00:15:58||2092-10-07
    +-38|-28098|-890374552|-8581765103969312768||-3283040.77|2656201211529.278320|true|industrial engineering|gabriella polk|2029-01-04 19:28:12|2014-02-05 04:57:17.37210387|2038-02-26
    +-67|-9676|-1889139541||-11999.87|-2509288.07|4834823806574.134766|false|linguistics|irene davidson|2039-06-08 03:04:50|2044-07-10 14:18:16.503028090|2094-11-08
    +-59|-15053|459269456|3542|24044.42|3949871.39|3217181341598.660156|true|kindergarten|mike nixon|2054-05-19 16:38:12|2053-10-27 15:11:39.341374749|1984-09-17
    +45|-9065|566646177|8553195689344991232|-8135.26|-1939444.49|1642913015191.496094|true|forestry|calvin ovid|2027-08-13 20:32:10|2034-04-21 19:49:54.457098155|2062-06-17
    +12|-4618|-1098379914|1789|36168.36|2705339.34|-3690347823231.547852|true|biology|holly johnson|2042-06-30 11:41:46|2018-07-12 01:39:37.64101956|2056-03-04
    +71|27905|-1755088362|8698055291501543424|-19752.00|33234.12|688042997736.531250|true|american history|oscar ovid||2053-09-30 17:29:18.665697992|2009-03-20
    +-71|-18503|597657990|296|-11362.53|-4728535.24|1779931909010.094727|true|religion|david falkner|2072-03-01 20:11:45|2037-09-21 12:33:06.143755779|2000-03-05
    +11|10940|1687784247|-9095689235523264512|4559.16|-3019879.00|-874527173245.560547|true|zync studies|bob king|2014-07-11 06:14:44|2077-11-30 06:12:58.616698981|2034-11-19
    +-50|5591||7998687089080467456||285146.00|3230421493464.880859|false|quiet hour|priscilla ellison||2042-01-18 02:32:10.14671907|2094-02-22
    +-90|19986|-1808960215|8160569434550403072|-36016.61|-2975819.86|4812496108620.009766|false|zync studies|ethan davidson|2040-08-07 19:44:11|2076-10-03 23:03:12.999374869|2062-08-07
    +4|-21009|-928013434|489|37854.16||4399713240319.478516|false|values clariffication|ulysses hernandez|2047-03-25 04:38:53|2054-05-31 05:02:46.363507150|2069-11-23
    +20|-23241|-1701492480|-9175038118837149696|-26613.47|1842617.05|-874710832768.563965|true|american history|jessica carson|2037-03-24 21:59:48|2075-10-04 10:31:47.885981681|2055-06-11
    +119|-6384|1187495452|8571268359622172672|-8449.84|-4234142.32||false|american history|xavier ellison|2058-04-04 16:57:08|2031-10-24 10:16:43.56250862|1981-11-13
    +26|23468|-2136052026|-7916510129632296960||-3375.58|-932636901856.333984|false|||2063-08-14 11:27:00|2029-05-07 11:16:44.211702565|2093-04-29
    +-43|24298|-1524554771|8323460620425330688|38854.35|-4288521.90|816149960555.847656|true|joggying|luke johnson|2039-12-24 23:32:35|2074-12-19 05:35:31.748853096|1994-05-03
    +|-11767|-938756287|346|-14372.50||2123388956108.485352|false|philosophy|fred steinbeck|2023-06-14 17:07:18|2013-08-25 12:57:38.526376049|2068-09-07
    +82|-6334|-564495517|3980||-3367097.22|-974002789414.605469|true|american history|sarah steinbeck|2077-09-25 01:00:46|2031-02-06 14:50:01.97188046|1986-09-18
    +-34|-13335|1568180994|-7707242953271500800|43312.27|-4190279.40|3157550021812.021484|true|education|alice underhill|2077-03-07 09:32:11|2061-12-11 17:47:50.656826729|2033-07-02
    +-41|10278|-922200749|1811|-5675.84|2126493.90||true|linguistics|ulysses polk|2021-09-26 02:58:42|2037-02-07 22:52:41.845445061|
    +-3|16134|923980398|2803|47958.63|-123069.85|3302684840602.587891|true|xylophone band|katie van buren||2017-11-28 19:30:26.747429591|
    +-48|-27284|-215703544|7370078518278397952|-44853.80|2971392.13|-2865549427171.991699|true|values clariffication|yuri thompson|2040-10-13 04:09:32|2042-06-15 18:08:24.614905000|1987-10-30
    +122|-21805|1543611951|7497276415392407552|-21005.13|2798310.44|-4898025473643.805664|false|american history|tom quirinius|2027-09-08 19:38:10|2073-02-25 17:53:54.931182325|1974-09-07
    +29|-4772|-2028355450|2323|-14463.60|-2063292.11|3614336014381.258789|false|quiet hour|quinn thompson|2040-10-17 05:30:18||2073-04-07
    +22|-23622|916057807|8467976965865799680|-45069.06|-3169540.55|-3121144891991.676270|false|american history|katie ellison|2039-06-07 19:13:41|2064-10-13 03:30:27.872003019|2064-06-14
    +54|25636|-1156193121|691|14845.29|2712830.87||false|industrial engineering|yuri johnson|2075-04-20 17:39:07|2036-11-16 16:14:15.788195860|2053-03-05
    +61||-1462331586|1914|-19390.54|-2888414.09|-1458422440761.921875|false|topology|gabriella xylophone|2043-05-11 03:31:25|2016-03-24 12:00:25.125315901|2067-01-30
    +-68|-27049|-941433219|6982145326341423104|-17847.79|2416407.91|3913372951778.839844|true|quiet hour|oscar white|2016-12-13 22:46:01|2042-12-23 09:34:08.712953951|2092-12-18
    +-14|-5357|-671853199|-9203804401302323200|-49229.73|3168248.28||false|linguistics|fred falkner|2045-09-30 02:09:20|2069-07-07 15:06:02.724854172|2093-12-04
    +-125|-23546|344239980|7823874904139849728|41579.55|1950776.66|1201402518499.143555|false|joggying|david hernandez|2034-11-18 20:10:51|2054-11-24 08:43:56.953788150|2105-09-20
    +54|26155|-532755480|7534145866886782976|6967.23|-3226525.02|-1855303775517.675781|true|debate|rachel van buren||2055-01-08 05:36:23.516296700|2012-04-21
    +-1|-14551|76381404|9085434340468473856|-35645.13|2319559.14|-308755508523.432617|true|quiet hour|rachel garcia|2017-01-24 05:00:43|2013-12-09 10:54:08.794663094|2052-11-16
    +-97|-3619|-1545388906|8579974641030365184|-21168.24|-2764375.21|-3347449615248.976562|false|values clariffication|bob ellison|2055-05-20 19:13:26|2033-04-21 06:38:32.498310577|2017-11-30
    +100|-6024|1723691683|8536948829863198720|47627.04|-1569764.42|-2866718883273.335938|false|zync studies||2054-06-22 01:40:38|2058-02-06 15:15:47.940520185|2072-11-26
    +126|5516|278643258|341|-11228.83|634413.34||true|history|xavier ichabod|2022-09-19 23:14:38|2079-08-22 14:14:01.57697301|2067-03-01
    +78|-10439|742866312|-9102482277760983040|17001.89|3890002.99|2756529687561.446289|true|history|victor zipper|2017-08-01 15:57:45|2070-05-02 05:20:55.632052613|2030-10-04
    +-103|32210|-1254129998|658|-29957.29|-2795444.37|75939180315.778320||mathematics|ulysses brown|2027-08-02 14:55:50||2081-08-17
    +80|-5089|-588547970||40137.08|4196447.38|2249064278951.813477|false|values clariffication||2044-10-13 20:35:22|2063-07-28 15:07:41.30831543|1990-09-22
    +-17|-14705|-887790938|2843|27083.98|-4094095.34|-4538477190343.535156|true|study skills|priscilla ichabod||2060-07-21 05:29:07.891389875|2020-08-08
    +41|-22910|1410516523|7584007864107778048|-26850.55|1924125.21|2728934571209.719727|true|education|yuri ovid|2069-09-25 12:15:22|2081-01-21 23:09:27.167408818|2105-06-13
    +53|-16247|-186764959|590|-43932.82|1488803.45|426645021297.744141|true|yard duty|xavier steinbeck|2052-08-30 02:52:56|2057-09-22 01:17:56.602173912|1977-06-25
    +124|-1067|-2146432765|8899122608190930944|-8534.65||3325424877285.349609|true|religion|nick van buren|2058-01-20 15:06:15|2077-04-04 20:32:46.931020333|2026-01-11
    +-36|27156|850625480|3588|42187.91|1811969.63|-3976647851697.982910|false|topology|zach allen|2014-07-18 20:02:05|2039-08-04 00:06:01.390839073|2058-05-02
    +104|10457|-1380191654|3609||4276263.85|-265494965739.847656|true|chemistry|wendy steinbeck|2076-10-08 03:34:56|2043-11-26 06:58:33.589491132|2082-02-15
    +111|-260|1372224352|3824|-10063.81|3381864.81|4651675165956.277344|false|education|tom underhill|2015-09-12 05:40:29|2024-06-13 07:25:09.53150568|1983-10-31
    +-41|276|-2124994385|7690986322714066944|10570.59|1504218.24|4721071679740.945312|false|undecided|ethan young|2057-12-06 16:20:41|2044-12-03 15:12:53.223286147|
    +-98|-4286|1074488452|7765456790394871808|34635.92|2380624.11|1160277570495.260742|false|industrial engineering|tom polk|2046-08-09 21:35:24|2063-02-09 19:10:28.968599555|1985-02-08
    +5||-500921094|-8649711322250362880|2739.41|2974096.30|-3103906430076.932617|false|debate|irene van buren|2078-11-04 19:08:18|2030-03-03 09:27:14.362039938|2065-12-20
    +-106|11895|-628446314|1948|-16324.00|-4498818.60|1427707877153.423828|false|debate|||1970-01-01 00:00:00.172894766|2080-10-14
    +86|-16390|-1918847735|-9101953184875757568|-11869.39|2819946.57|597670762650.892578|true|geology|priscilla miller|2048-11-20 13:09:10|2027-04-22 12:21:15.759028066|2038-12-26
    +52|-30492|2031604236|2463|38249.67||2340697225707.053711|false|nap time|quinn johnson|2027-10-15 08:46:59|2066-12-22 01:05:49.826136001|2088-10-12
    +-69|-13410|-738157651|1813|20147.37|4298419.86|-471086287570.549805|false|kindergarten|david allen||2078-02-24 23:37:09.324361343|2023-05-09
    +50|-12422|-1266138408|7054271419461812224|33076.17|4726793.74|-1825975796516.701172|false|study skills|gabriella steinbeck|2034-09-27 07:53:11|2067-05-04 08:00:04.76094355|2045-08-30
    +119|6913|1836499981|7548958830580563968|-789.80|-4853521.51|-2520439759952.566406|false|education|yuri davidson|2037-12-20 12:22:00|2060-03-04 05:27:06.411478902|
    +-68|22588|2084666529|-9206329156028112896|-46857.55|4518647.92|3514007061626.251953|false|industrial engineering|mike brown|2025-09-06 20:02:42|2017-06-08 12:00:40.56190634|1976-03-22
    +30|-26180|1522208504|2637|24228.24|4916304.14|2690767997858.607422|false|study skills|fred hernandez|2018-08-31 02:11:20|2034-06-17 01:40:03.264737765|2005-11-23
    +-106|-25689|-693249555|-7661250850555633664|-24825.24|1483670.11|-4990192589949.166992|true|geology|nick king|2019-01-30 15:29:01|2016-03-23 21:48:37.760268674|2081-06-09
    +113|24019|899810881|664|44245.25|4097479.70|294259545397.430664|true|biology|quinn garcia|2017-11-03 16:12:54|2065-09-09 02:55:30.992457875|2044-04-01
    +-73|-7873||2487|-21944.26|-1353294.67|3144741114101.847656|true|american history|sarah quirinius|2068-12-01 14:02:38|2025-09-28 09:32:04.373329529|1984-03-30
    +-29|-4211|-1626062014|8221561626658881536|-17167.97|2376404.62|-2396766266715.104492|true|religion|jessica allen|2076-10-17 13:16:32|2041-07-29 04:57:59.28367872|2052-06-27
    +76|-19545|1765874562|8169878743136043008|4737.74|2522751.77|4492331567972.839844|true|undecided|mike ovid|2025-02-11 02:25:59|2057-07-18 07:47:20.80373888|2035-05-28
    +120|15578|1668094749|6927260280037097472|-12442.12|-3625208.20|-2505949310240.964844|true|values clariffication|luke carson|2060-08-28 06:39:49|2042-08-21 19:21:53.22689204|2036-04-04
    +-121|-32403|-884796655|342|43804.75||3086918973716.556641|true|philosophy|oscar robinson|2042-10-22 23:52:01|2027-05-27 10:44:21.656527878|2057-06-19
    +-2|-27807|1809795770|-7501803640821456896|-31728.95|2217700.27|1303564912300.209961|true|linguistics|david ichabod|2058-03-12 04:48:33|2026-10-07 09:37:11.609376049|2096-11-02
    +39|-2410|1134416796|2745|32844.17|-1496658.25|-634697885935.402344|false|opthamology|quinn thompson||2043-04-08 21:14:44.886763154|1974-03-31
    +57|-3449|-1038565721|677|-32939.72|3205392.52|-3723568349288.726074|true|yard duty|wendy laertes|2039-08-20 17:31:07|2043-12-22 07:02:29.625016869|2068-08-18
    +-52|-19028|-2081809883|8435912708683087872|-49142.74|-1903864.82|3512222649259.086914|false|mathematics|gabriella brown|2040-06-21 05:21:16|2058-07-03 23:16:43.395150800|2059-05-01
    +-123|12674|1817671655|7412924364686458880|22076.92|1732310.78|-198468750423.259766|true|values clariffication|yuri van buren|2062-02-20 04:56:42|2074-11-24 19:41:00.713104566|2003-07-31
    +-76|-7533|1332181668|3563|16233.53||3796582343071.613281|false|yard duty|ulysses laertes|2029-10-24 07:37:56|2071-10-19 15:34:38.872812372|2071-04-18
    +-119|-14053|8040290||9147.84|-4731232.91|1812411477716.757812|false|values clariffication|holly ichabod|2031-01-05 20:55:56|2050-12-24 02:26:09.954349620|2097-09-19
    +110|-17426||7153922334283776000|10695.41|-4157898.82|-3205627224037.357422|true|yard duty|calvin ichabod|2022-06-25 07:08:22|2061-01-13 03:07:32.453720097|2066-09-03
    +-28||718692886|8849475396952514560|-22341.30|-1839215.40|-2738602396779.903320|false|opthamology|victor white|2030-06-25 00:58:04|2061-03-21 18:52:13.283910292|2095-01-04
    +90|-16129|-2007662579|2977|15304.02||-4049128118326.663574|false||wendy quirinius|2080-06-24 16:38:20|2017-06-18 02:28:45.172735440|2050-09-13
    +96|1409|-1259611508|-7910019233726242816|-38530.51|-44517.83|-3803225081574.588379|true|industrial engineering|calvin brown|2017-03-12 03:01:53|2033-08-29 02:29:46.117205169|2010-04-10
    +117|-32688|144428297|2835|-20190.35|3425255.76|1239390523307.939453|false|opthamology|rachel falkner|2056-11-05 14:04:47|2075-05-31 12:50:08.854908788|2084-04-26
    +55|999|1281277970|2335|-2914.82|||false|opthamology|fred young|2072-08-24 05:04:35|2058-05-10 01:43:32.344081382|2029-07-31
    +41|28165|1069486136||-37952.82|-276339.96|3109359968514.944336|false|opthamology|victor ichabod|2069-02-08 14:25:38|2076-05-15 06:44:30.554693842|2053-03-25
    +-86|23850|-601946913|2515|10326.69|4649531.05|2268855975314.246094|false|topology|ulysses robinson|2013-04-25 03:23:27|2056-06-03 12:33:31.602556689|1984-05-06
    +-101|718|386741352|-7617860842651017216|-42832.01|2064169.43|-4943806409743.167969|false|industrial engineering|sarah ichabod|2057-03-27 09:22:04|2064-01-08 17:40:10.787568720|2020-01-01
    +-127|-10662|648935848|-7637755520917741568|-33941.20|1359686.42|1579030228803.083984|true|chemistry|luke laertes|2016-12-23 11:45:20|2036-05-11 12:47:07.733851638|2003-04-13
    +80|-1223|92834720|2647|-36870.03|1719246.27|2540178498144.847656|true|nap time|bob falkner|2038-06-08 07:54:27|2042-11-05 13:52:50.588354549|
    +-3|22320|1343581455|707|15471.72|-4962575.43|-1611227671309.833984|false|study skills|mike robinson|2078-04-16 04:19:35||2048-11-11
    +||-4943292|8856674723376668672|2164.62|893829.85|-631328089657.294922|false|opthamology|oscar quirinius|2027-03-09 19:18:57|2054-01-23 04:24:15.292126981|2060-03-20
    +-50|2616|977624089|7857878068300898304|-30457.66|-1477221.11|859058050811.647461|true|american history|ulysses ellison|2035-05-02 05:58:22|2050-12-28 10:26:48.409801540|
    +3|-17916|-191704948|-8887058200926093312|35825.27|4647161.69|2477546519048.629883|true|mathematics|wendy carson|2027-09-25 05:34:03|2032-03-05 14:53:42.83179281|2024-12-19
    +100|-5919|-835107230|108|-37662.39|-3446860.49|4914447452945.439453|false|kindergarten|sarah carson|2031-07-27 00:13:59|2066-12-30 00:14:45.43797107|2058-07-30
    +37|-24696|314232856|2762|-9928.97|-2740875.06|3528496539480.343750|false|geology|fred thompson|2032-05-15 14:11:38|2057-07-08 21:29:58.104912394|2091-09-09
    +27|-8398|-1583445177|3622|32876.52|285737.42|-4676053735656.750977|true|undecided|luke xylophone|2071-11-07 15:31:50|2021-08-10 08:22:04.527867145|2086-02-10
    +|-14644|-2133145181|868|-5084.93|-77615.10|4647718670978.201172|true|chemistry|yuri thompson|2030-06-15 16:57:13|2063-01-20 09:04:49.362142228|
    +36|-7046|843282593|138|-1437.75|1530832.04||true|biology|zach van buren|2049-05-13 12:51:19||2039-05-11
    +31|-10258|-1009249550|1786|-45853.99|-2996009.61|-1270664267650.971680|true|zync studies|calvin laertes|2054-10-23 17:10:03|2054-10-22 11:44:42.793294685|2064-05-02
    +||-236700442|9116137265342169088|13586.84|-3397114.60|1665543214302.829102|true|biology|holly carson|2076-12-15 12:30:43|2067-07-10 06:37:03.526969971|2038-06-13
    +-8|-25988|1447438548|7955126053367119872|-37404.36|-4278525.05|-1539681145987.756836|true|forestry|david thompson|2056-07-24 05:41:13|2038-11-27 05:16:00.826863713|1976-09-08
    +-93|4747|-201554470|491||-4736059.79|-4074148068973.322266|false|religion|luke brown|2052-04-12 11:55:51||2019-08-20
    +18|-26064|-807242371|-7778829032042790912|-497.18|3018359.10|4165268868128.960938|false|history|quinn xylophone|2064-08-04 14:39:21|2049-07-11 03:38:36.188505486|1979-06-14
    +-2|28921|-1599905147|-7753051494275432448|38746.13|1924853.86|2728705276812.348633|false|chemistry|mike miller|2063-11-22 05:40:00|2040-04-01 02:20:42.508563674|1999-07-08
    +|-26946|-329695030|8962097525980225536|47503.36|-817384.58|955232900660.511719|true|zync studies|holly van buren|2020-01-09 00:26:09|2033-08-15 04:18:59.595230815|2066-06-06
    +0|-2835|1968813171|8163948965373386752|8908.65|-3820148.18|-481479853467.270508|false||sarah miller||2068-01-02 01:54:57.346312106|2075-09-25
    +114|20265|669484010|1145|13147.10|-1843756.18|554432518470.839844|false||yuri ovid|2039-05-04 13:47:10|2038-08-21 17:57:56.403669763|2081-07-12
    +|28775|232405034|-8438554249514491904|46640.35|-4159360.82|-255361794196.424805||education|quinn robinson|2026-01-11 21:41:33|2040-12-26 20:16:11.824788390|2024-05-19
    +111|22074|-853606287|522|35504.72|1790821.65|38856848277.219727|true|american history|alice van buren|2054-05-22 23:30:43|2046-11-03 15:13:41.781244421|2079-09-30
    +55|4319|-524189419|1785|34553.22|-1096127.27|4436479409217.000000|false|biology|xavier van buren|2079-08-25 13:19:06|2080-03-03 09:31:40.884489078|1979-09-24
    +51|-800|564366133|1545|-31100.82|-1936194.76|1798080391637.881836|false|undecided|alice xylophone|2065-08-30 18:15:20|2055-12-07 19:37:09.85869965|2078-10-08
    +107|11159|-346607939|999|-24238.72|794247.13|835410305018.271484|true|forestry||2072-11-24 12:45:43|2027-01-26 04:52:45.940702976|2001-01-27
    +-116|-31182|-54793232|1941|-33327.99|-4081209.43|-4789172370818.662109|true|wind surfing|jessica van buren|2062-10-28 14:12:34|2047-11-19 19:37:15.731253997|1983-02-23
    +-10|-32371|1333214263||-32279.82|4486493.62|4879631964298.525391|true|undecided|quinn falkner|2045-05-25 12:09:59|2068-07-04 02:37:10.599103732|2089-02-09
    +80|21377|-267554590|7454442625055145984|8701.65|-905741.94|1537464945669.313477|false|joggying|david van buren|2077-05-12 07:35:39||2061-02-10
    +-104|-3190|197056787|3510|-22360.28|4523037.34|3240433039663.095703|false|xylophone band|xavier white|2051-07-06 22:05:57|2051-01-02 05:43:39.718899624|1996-03-18
    +-64|6163|-1602792666|2373|16736.31|-3740028.62|3690526967552.294922|true|joggying|yuri brown|2040-02-10 20:14:48|2080-02-11 07:48:36.902436157|2000-11-17
    +-30|-24670|1701817607|-8127494999848919040|-13803.97|3963022.39|1474152640583.561523|true|geology|tom hernandez|2079-07-30 04:14:31|2057-05-04 01:14:00.609761058|1974-09-16
    +-74|-23109|1346627771|1643|-43168.85|3155689.66|-4448781974974.628906|true|forestry||2039-08-04 16:00:45|2028-08-01 19:19:13.322219484|
    +118|-23389|-370093295|-7819437864839495680|-26609.73|-4237190.22||false|kindergarten|bob thompson|2019-09-22 15:35:21|2037-04-28 10:43:14.893096143|2021-09-12
    +69|24253|60847311|-7822452149325094912|-40655.11|-3881485.82|-1314247053468.484863|false|debate|oscar young|2067-05-18 04:05:27|2039-10-06 08:12:08.884589651|1984-06-30
    +-5|-32000|-177025818|7411793502161182720||2485530.29|1594683877449.583008|true|opthamology|yuri young|2035-11-20 23:20:18|2078-01-19 12:17:21.914800889|2015-02-06
    +-35|7474|-1676261015|2274|38942.74|-37023.70|3451540509748.127930|true|opthamology|katie polk|2043-05-05 01:41:36|2044-09-13 08:37:07.823303166|1973-01-15
    +-102||-714270951|8783241818558193664|-24442.85|50032.07|3379969757973.632812|false|topology|calvin xylophone|2049-08-05 08:30:47|2065-10-10 06:49:24.92342328|2078-07-28
    +84|-18485|345556325|8316336224427483136||4778.81|-213703503553.759766|true|geology|irene laertes|2027-03-23 13:59:49|2068-08-29 13:45:16.22118413|2004-10-29
    +57|-19535|-1484033125|-7669169138124275712|5132.56|-3812127.71|-2880380214433.984375|false|values clariffication|xavier ellison|2013-05-28 14:20:54||2032-11-10
    +19|-16815|1440427914|2984|-33347.21|-2217545.87|-1622352479297.113281||xylophone band|katie garcia|2037-08-01 18:21:41|2074-09-05 16:07:02.893370421|2050-05-01
    +-42||-1201785350|-7772064021830574080|-19385.56|-2417269.98|4254639589730.294922|true|wind surfing|quinn thompson|2056-10-03 11:35:20|2050-10-14 15:57:47.94057206|1994-04-07
    +-26|-20787|-2066134281|3397|38532.91|-2052731.66|-2212037711168.371094|false|xylophone band|alice zipper|2081-02-22 01:21:53|2066-09-15 08:59:43.712981923|2068-06-30
    +-27|-20093|-226635871||39730.55|2920764.76|3736425396480.705078||american history|alice davidson|2063-06-30 00:25:29|2039-10-04 08:34:32.485344607|
    +-52|17720|2134433675|8523972434954510336|-41237.62|713755.68|-2484484479175.712402|true|education|irene ovid|2035-11-11 02:58:59|2036-10-18 01:59:34.57776882|2071-03-20
    +-28|28089|260463232|-7127548949860818944|44331.03|-3844217.64|-1326537974723.827148||undecided|ethan davidson|2041-01-01 12:36:24|2023-11-30 11:12:45.269637203|2029-01-15
    +3|6587|-916495008|8286706213485297664||-454411.58||true|xylophone band|priscilla polk|2066-01-13 23:59:59|2028-04-05 20:31:53.571987516|2031-08-14
    +-96|-7065|1102069050|3147|-5252.56|2525210.28|-2140534616823.686035|false|wind surfing|oscar polk|2078-12-19 18:51:30|2024-01-19 18:54:53.308875328|2070-05-01
    +-87|-134|-1289501869|-7536330682873937920|-32963.40|4026456.79|-4233812360710.160645|false|american history|irene king|2076-12-17 13:17:28|2026-03-07 02:44:03.954816308|2039-03-16
    +-88|-29977||-7115054815375073280|-49346.30|-4958976.45|-2795424166054.196777|false|mathematics|zach steinbeck|2033-10-10 03:50:46|2072-01-30 01:12:19.980530173|1988-11-14
    +-54|11077|-235238928|-7319315187617587200|-21327.60|3102506.04|3723093064701.678711|false|biology|quinn hernandez|2050-07-10 07:39:25|2067-01-01 12:42:39.897268043|1977-03-31
    +-127|14408|1390704286|1099|-26406.94|4860414.94|4855359545894.408203|false|quiet hour|mike miller|2057-01-23 19:19:39|2064-10-13 08:42:40.250692511|2030-08-25
    +-110||1310360849|-8989473881707921408|29776.70|4921339.21|-1711490983970.094238|false|american history|mike van buren|2017-06-03 10:24:21|2017-06-10 20:23:52.816880923|2042-06-26
    +-108|6226|-912429611|2816|30162.78|682391.80|618292737301.046875|false|debate|sarah steinbeck|2059-03-16 19:54:08|2015-05-02 04:32:12.949465541|2008-06-28
    +60|18845|-1369253050|-6986178228432322560|-35269.73|2961506.86|1006481369666.687500|false||katie white|2022-02-26 02:53:55|2031-08-28 00:04:15.166189356|2054-08-15
    +46||-938762477|-7759425383684849664|46392.36|1976120.50|4269437008221.785156|false|mathematics|ethan steinbeck|2061-01-29 15:15:46|2064-07-10 10:33:03.635139975|1978-09-04
    +-71||268888160|-7893577088764174336|-35474.68|-4088895.16|-617279427364.288086|false|nap time|bob hernandez||2040-02-11 11:11:34.256681234|2061-05-04
    +91|22232||8091421389575282688|9502.48|-3837325.10|-1866384122906.517090|false|zync studies|holly young|2043-03-07 03:52:18|2033-07-28 01:11:14.500349492|2010-08-03
    +77|-6884|-624029057|-7409653086454030336|39017.19|-2245679.39|-997439424357.757812|false|debate|irene robinson|2077-03-18 03:11:00|2040-04-20 17:23:11.66448920|2025-08-31
    +-101|12411|-1940205653|7348598907182800896|43885.62|-4414072.29|-1555511953931.052246|false|american history|david xylophone|2029-10-21 01:48:10|2043-08-31 14:12:30.675070864|2085-10-24
    +-57|11804|-496915240|-7362189611124563968|-8203.40|3975044.25|1092860346014.425781|false|chemistry|oscar allen|2042-05-21 21:47:45|2031-01-19 14:17:45.573707182|1978-09-30
    +59|27945|474795096|||-1041376.93|1049208781647.408203|true|zync studies|mike young|2042-08-04 13:26:56|2076-09-18 22:30:21.635931282|2100-08-21
    +|6490|-1819075185|2465|271.01|-3315662.78|178512873486.944336|true|philosophy|luke miller|2058-11-13 12:26:19|2055-04-29 01:32:52.338899070|2072-05-20
    +59|-13144|330302407|350|4198.95|-3293708.78|-837074147937.393555|false||calvin davidson|2048-05-03 07:09:22|2016-02-14 09:35:08.980648052|2050-12-28
    +-58|-8895|1895751360|2619|44281.94|-3137628.74||false|topology|ethan underhill|2067-05-09 03:59:27|2070-01-06 00:22:49.266686299|1988-03-03
    +41|-5283|-1322736153|3722|33698.34|4876900.77|2424764462849.715820|true|wind surfing|jessica polk|2016-04-22 18:45:33|2078-05-27 02:08:56.801166778|1997-08-03
    +56|-22608|338805871|898|4188.81|4744554.21|730519058324.112305||mathematics|priscilla falkner|2069-12-21 21:41:25||2070-01-08
    +-56|10702|-1552053883|782|-38874.21|-1017367.57|4561020549763.021484|true|religion|calvin xylophone|2071-01-01 17:03:06|2016-11-14 05:12:35.708167457|2080-10-11
    +17|-9585|-71433796|1780||509725.97|-1030876895734.564941|false|religion|mike johnson||2077-05-06 09:41:04.29662719|1978-11-15
    +110|-24095|-1655396452|2186|-45152.50|3996472.52|888655102684.872070|false|quiet hour|mike hernandez|2065-12-16 23:03:21|2065-07-26 03:28:40.392848789|1991-03-29
    +52|21673|-1066775085|-6921654334727036928|-9981.07|-1650150.10|-4371310034780.968750|false|history|katie zipper|2031-09-27 21:23:48|2080-08-25 23:49:27.604332416|2069-04-08
    +-113|20052|-1447140800|4020|-22503.13||-4288708971525.778809||wind surfing|mike underhill|2079-05-03 00:37:57|2078-03-29 14:33:49.853013719|2043-01-06
    +-61|28000|-178568841|8325227661920133120|-22983.25|1809174.68|-2297090489976.945312|false|history|nick quirinius|2069-09-27 08:08:25|2043-04-20 02:53:33.962471217|2044-06-19
    +102|-31458|-1933192293|-7036607470351654912|-21380.15|1896367.69|1605659965343.057617|true||rachel white|2015-06-12 13:38:42|2068-03-09 14:21:05.750561437|2099-06-14
    +-8|-25417|1961954939|7304839835188609024|-41942.73|191389.71|-4592978910553.853516|false|undecided|nick ovid|2035-11-09 10:25:30|2077-04-15 14:29:55.59700199|2010-11-12
    +41|-10569|-745678338|8792059919353348096|-14404.47|-906177.24|326664391884.108398|false|philosophy|priscilla steinbeck|2034-04-20 09:29:51|2042-10-01 01:12:43.143080614|2048-10-17
    +-41|4233|618321042|-8856821118526734336|-27396.30|4504910.53|180650253126.657227||debate|tom xylophone|2027-02-11 05:29:58|2066-07-09 11:19:47.657330816|1995-12-16
    +-93|18820|825677248|8720504651219001344|9437.05|-4519097.40|-506526954412.187500|true|philosophy|alice miller||2077-05-05 12:20:54.799217277|2001-02-16
    +33|-3103|371383749|1055|-43829.05|-3580973.49|3029406193241.257812|true|wind surfing|jessica quirinius|2026-05-11 10:04:20|2074-01-01 07:56:24.948489929|2096-02-29
    +-95|26529|-223311809|1368|-19919.52|2770068.53|-2537836240149.431641|false|debate|gabriella miller|2040-03-14 14:51:41|2037-09-24 04:28:20.146749934|2102-01-22
    +79|-28084|-1974972123|8736061027343859712|-28922.94|-2498365.96|-985441433336.760254|false||rachel laertes|2032-06-23 14:29:26|2023-01-25 12:10:28.866537625|2105-06-23
    +70|6781|2125311222|7919597361814577152|47154.09|-1192322.16|-850435843660.583984|false|religion|alice polk||2065-10-31 11:02:12.417523639|
    +-1|22463|867587289|7381659098423926784|-29302.26|-2211405.89|1156004227362.394531|false|study skills|alice white|2038-06-11 11:52:35|2026-02-06 12:13:27.232943870|2090-01-01
    +15|7392|869288953|8731960288562044928|41942.98|3624760.68||false|zync studies|alice miller|2058-09-06 21:08:50|2060-02-22 03:34:09.543087589|2076-07-25
    +-57|-24942|824743780|-7594824008626372608|35489.13|1006528.68|-4120539026250.966797|false|geology|priscilla falkner|2066-03-31 07:55:39|2053-11-01 10:52:13.944770605|2080-12-02
    +-26|-14810|-1407817977|-9178166810751909888|37075.67|-4876175.79|2103363485344.971680||chemistry|tom quirinius|2026-05-24 15:05:33|2014-12-24 04:57:37.782711517|2000-06-22
    +-70|6484|-385247581|3083||-2073947.26|890806455827.118164|true|values clariffication|ethan steinbeck|2018-06-04 10:12:46||2055-12-17
    +79|-15946|-1078397698|-8948335470186373120|-22913.97|227203.97|4075094455625.994141|true|industrial engineering|david robinson|2033-07-28 05:43:26|2020-05-29 18:18:23.223813235|1982-05-04
    +-75|-17873|-837503491|2569|6361.99|1161789.65|-1726978053243.932617|true|values clariffication|quinn robinson|2045-04-18 18:19:14|2021-01-27 02:30:40.164402829|2017-01-10
    +96||1660088606|823|-28084.37|-3275847.50|2762686693256.810547|false|american history|ulysses nixon|2070-08-11 19:10:10|2016-09-07 01:20:11.409060890|
    +-113|-30515|-922875124|259|4946.14|284363.03|-4858481689197.126953|false|kindergarten|fred nixon|2023-06-28 22:37:13|2049-06-26 22:24:15.711126823|2069-02-06
    +49|3186|1636364987|8461498293348065280|-25371.38|-1184912.70|-993292644578.538574||education|alice white|2058-11-21 02:58:56|2065-03-02 04:53:08.176223587|1994-05-02
    +63|150|1783034168|-8961059046745669632|||-1661232720992.700195|true|topology|quinn young|2038-09-04 07:44:08|2047-01-27 11:44:14.355445111|2043-02-21
    +74|-21648|-1111937842|-8607195685207408640|-38757.30|4280929.21|-3715598460104.345703|true|joggying|irene white|2017-10-30 12:48:38|2013-03-25 15:32:17.128329365|2017-10-02
    +79|3251|-125419186|-8754966081778565120|32512.77|1838279.26|-1177230386696.095215|true||zach laertes|2059-05-28 03:36:52|2049-09-18 13:52:36.954456644|2013-05-30
    +96|-10126|-41864614|-8418913260807217152|-41581.70|3132782.56|-4989436752632.237305|true|opthamology|victor king|2025-08-12 00:24:07|2025-04-07 14:05:54.461436636|1987-03-18
    +|-6474|706823078|-8877053610728161280|-32523.08|2431805.10|959431399353.079102|true|joggying|calvin ovid|2024-11-30 09:52:29|2026-10-20 13:44:56.808161722|2038-08-30
    +-85|24858|-1062159435|-6935548339131138048|38973.86|-347553.95|3408594399945.599609|false|biology|oscar zipper|2026-05-18 20:07:36|2018-09-17 07:34:48.856373956|2044-03-13
    +-44|30883|1452244326|-8219876839318716416|24860.92|-3405708.68|-3460325348420.880859|true|education|wendy quirinius|2038-08-09 15:24:29|2054-12-15 15:20:18.843218245|2061-01-18
    +88|-9076|239078089|1132|28538.60|-735298.99|-4690879292129.750000|true|wind surfing|zach quirinius|2071-11-14 00:07:27|2047-06-15 18:39:40.150298534|2079-04-15
    +48|23320|-1948257321|1337|-43790.15|3928891.37|3704624417356.896484|false|joggying|tom hernandez|2053-10-13 18:39:56|2026-03-31 20:18:01.770207236|2034-08-29
    +-98|7197|492120544|1341|-2808.88|-686072.73|-803606583331.628418|true|education|tom xylophone|2015-05-08 21:19:10|2077-10-24 08:05:04.192595595|
    +72|7058|-1563676282|976|-3004.90|1860479.52|-706916831724.306641|true|joggying|victor polk|||2058-01-01
    +38|-14661|195281533|-7557017910095650816|-9383.79|-2348124.75|4636426753661.630859|true|mathematics|tom thompson|2067-05-15 15:21:33|2068-10-26 13:18:32.366613022|1975-07-07
    +16|-28061|-292588406||23350.65|4918613.06|-3364353490553.036133|true|nap time|calvin steinbeck|2054-01-11 20:28:18|2037-02-15 00:30:35.900105726|1970-08-29
    +-104|19636|-1344287228|-8683802826440105984|8738.40|-1978806.64|4981312863247.886719|false|biology|holly davidson|2017-11-02 22:33:43|2061-05-26 11:57:49.809511168|1993-04-14
    +-59|-2113|-909024258|1845|41057.97|213462.82|-323293361486.882812|false|xylophone band|ethan carson|2049-05-23 19:46:12|2079-02-24 16:42:08.448187732|
    +70|-13309|1173098061|1965|1703.69|2429715.56|-2779513442254.710938|false|joggying|calvin brown|2079-05-18 11:45:16|2018-04-22 08:38:53.938611457|2020-02-01
    +19|-17269|-1091003492|-8104684579106914304|-1914.23|281948.98|2761939531515.835938|true|kindergarten|holly robinson|2044-05-20 18:29:37|2037-05-21 10:21:17.589154522|2003-03-13
    +-96||1768399622|1835|5444.81||-137609076468.194336|false|xylophone band|ulysses king|2031-03-19 20:16:45|2034-05-25 03:38:47.816839184|2064-05-20
    +-28|10916|849859032|7345991518378442752|-3117.17|1217133.63|-3849320444967.645508|true|linguistics|irene quirinius|2060-08-22 17:37:02|2013-09-04 14:35:51.471949124|2092-02-05
    +10|19458|900992177|3212|36450.28|1766157.28|-1882794242083.676270|true|yard duty|fred king|2042-03-05 10:51:23|2075-06-04 16:36:19.586166561|2094-02-10
    +52|-18283|-1969751342|-7081500255163727872|36032.40|4272798.19|-3071209073548.714844|false|opthamology|victor ichabod|2034-03-30 05:28:25|2026-11-18 10:38:29.612055317|2045-12-20
    +125|16516|161210995|1074|-24350.82|2460973.22|921740273811.693359|true|linguistics|priscilla thompson|2024-07-31 16:19:23|2052-06-19 19:48:51.128000435|2031-11-09
    +62|30936|1321678350|8372588378498777088|1059.15|-3300550.22|3839852559580.664062||study skills|ulysses ovid|2049-11-14 21:22:02|2034-12-30 19:24:13.874541216|2037-11-28
    +-56|-23387|-1811563127|-7593363318079610880|-17859.26|-4870696.94|-1931143655238.862793|false|philosophy|jessica hernandez|2017-09-26 18:25:13|2023-12-17 02:12:59.770265368|2026-03-19
    +97|10217|1338047392|-7451660755269853184||587440.58|3232315873104.952148|true|undecided|victor ovid|2041-06-18 17:20:25|2029-12-16 14:12:32.298550349|1987-03-13
    +50|-12085|-1954890941|1983|-23284.45|129671.37|-2210750828357.674805|false|values clariffication|gabriella van buren|2055-07-03 23:20:38|2018-04-10 00:19:11.305802202|2103-07-27
    +52|-13805|415234946|8514851182589771776|-39177.49|3119458.07|-4784612318346.788086|false|forestry|bob brown|2019-06-19 22:27:39|2065-11-12 11:35:37.144557823|2041-11-16
    +104|||1864|13318.31|-3863136.50|-4419006380365.076172|false|joggying|luke steinbeck|2057-07-21 14:03:34|2026-03-06 18:45:53.361673899|2031-03-23
    +-81|25986|-1643714866|8463868417649524736|1656.77|-1600629.58|-4804769067544.612305|true|religion|luke ovid|2019-02-21 09:30:51|2051-08-22 00:41:24.173555438|2041-04-15
    +114|-28840|1335803002|3094|-22269.41|1188501.65|-2026377024912.235840|true|mathematics|david young|2054-03-31 08:13:24|2037-11-06 13:20:44.525490319|2093-08-21
    +28|27340|-1117019030|-8858063395050110976|17746.57|-370973.48|3665311301420.927734|true|joggying|calvin xylophone|2016-06-21 09:03:55|2038-11-18 02:43:13.946089513|1996-04-09
    +87|6875|-980869630|1981|-8269.53|-680305.04|-2046202768950.067383|false|forestry|oscar falkner|2054-03-28 23:22:20|2057-06-06 19:13:12.719272020|1984-12-16
    +-38|19894||-8140349174954893312|42897.60|-895172.87|-2612690969145.852051|false|religion|irene ichabod|2029-06-18 22:32:24|2069-05-05 07:02:12.743691084|1993-05-09
    +-94|9948|-455114104|-7041362811802148864|35927.66|2125413.96|928631125022.398438|false|american history|sarah zipper|2075-08-11 09:15:18|2025-01-16 11:03:58.820006581|2047-04-09
    +90||1709983738|8972161729142095872|27727.80|3584555.68|2487331275679.380859|true|kindergarten|wendy zipper|2075-07-22 03:37:27|2070-11-18 17:07:35.641105448|2005-06-22
    +-55|-4877||7989119273552158720|49881.20|-729941.08|2604782698166.197266|true|education|mike van buren||2059-09-21 17

    <TRUNCATED>
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10752: Revert HIVE-5193 (Aihua Xu via Chaoyu Tang)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5fdf143
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5fdf143
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5fdf143

    Branch: refs/heads/hbase-metastore
    Commit: d5fdf143d29344b6728e1489937a87e073aa9894
    Parents: 8aaf3bd
    Author: ctang <ctang@cloudera.com>
    Authored: Tue Jun 2 15:24:20 2015 -0400
    Committer: ctang <ctang@cloudera.com>
    Committed: Tue Jun 2 15:24:36 2015 -0400

    ----------------------------------------------------------------------
      .../apache/hive/hcatalog/pig/HCatLoader.java | 8 ----
      .../hive/hcatalog/pig/TestHCatLoader.java | 44 --------------------
      2 files changed, 52 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdf143/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
    index c951847..0685790 100644
    --- a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
    +++ b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
    @@ -19,7 +19,6 @@
      package org.apache.hive.hcatalog.pig;

      import java.io.IOException;
    -import java.util.ArrayList;
      import java.util.Enumeration;
      import java.util.HashMap;
      import java.util.List;
    @@ -32,7 +31,6 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience;
      import org.apache.hadoop.hive.common.classification.InterfaceStability;
      import org.apache.hadoop.hive.metastore.api.FieldSchema;
      import org.apache.hadoop.hive.ql.metadata.Table;
    -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
      import org.apache.hadoop.mapreduce.InputFormat;
      import org.apache.hadoop.mapreduce.Job;
      import org.apache.hadoop.security.Credentials;
    @@ -163,12 +161,6 @@ public class HCatLoader extends HCatBaseLoader {
          if (requiredFieldsInfo != null) {
            // convert to hcatschema and pass to HCatInputFormat
            try {
    - //push down projections to columnar store works for RCFile and ORCFile
    - ArrayList<Integer> list = new ArrayList<Integer>(requiredFieldsInfo.getFields().size());
    - for (RequiredField rf : requiredFieldsInfo.getFields()) {
    - list.add(rf.getIndex());
    - }
    - ColumnProjectionUtils.appendReadColumns(job.getConfiguration(), list);
              outputSchema = phutil.getHCatSchema(requiredFieldsInfo.getFields(), signature, this.getClass());
              HCatInputFormat.setOutputSchema(job, outputSchema);
            } catch (Exception e) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdf143/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    index fc18a3b..45a219c 100644
    --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    @@ -19,8 +19,6 @@
      package org.apache.hive.hcatalog.pig;

      import java.io.File;
    -import java.io.FileWriter;
    -import java.io.PrintWriter;
      import java.io.IOException;
      import java.io.RandomAccessFile;
      import java.sql.Date;
    @@ -36,10 +34,7 @@ import java.util.Properties;
      import java.util.Set;

      import org.apache.commons.io.FileUtils;
    -
    -import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.FileUtil;
    -import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.hive.cli.CliSessionState;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.ql.CommandNeedRetryException;
    @@ -49,7 +44,6 @@ import org.apache.hadoop.hive.ql.io.IOConstants;
      import org.apache.hadoop.hive.ql.io.StorageFormats;
      import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
      import org.apache.hadoop.hive.ql.session.SessionState;
    -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
      import org.apache.hadoop.mapreduce.Job;

      import org.apache.hadoop.util.Shell;
    @@ -66,10 +60,6 @@ import org.apache.pig.data.DataType;
      import org.apache.pig.data.Tuple;
      import org.apache.pig.impl.logicalLayer.schema.Schema;
      import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
    -import org.apache.pig.PigRunner;
    -import org.apache.pig.tools.pigstats.OutputStats;
    -import org.apache.pig.tools.pigstats.PigStats;
    -
      import org.joda.time.DateTime;

      import org.junit.After;
    @@ -492,40 +482,6 @@ public class TestHCatLoader {
        }

        @Test
    - public void testColumnarStorePushdown() throws Exception {
    - assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    - String PIGOUTPUT_DIR = TEST_DATA_DIR+ "/colpushdownop";
    - String PIG_FILE = "test.pig";
    - String expectedCols = "0,1";
    - PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
    - w.println("A = load '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
    - w.println("B = foreach A generate name,studentid;");
    - w.println("C = filter B by name is not null;");
    - w.println("store C into '" + PIGOUTPUT_DIR + "' using PigStorage();");
    - w.close();
    -
    - try {
    - String[] args = { "-x", "local", PIG_FILE };
    - PigStats stats = PigRunner.run(args, null);
    - //Pig script was successful
    - assertTrue(stats.isSuccessful());
    - //Single MapReduce job is launched
    - OutputStats outstats = stats.getOutputStats().get(0);
    - assertTrue(outstats!= null);
    - assertEquals(expectedCols,outstats.getConf()
    - .get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR));
    - //delete output file on exit
    - FileSystem fs = FileSystem.get(outstats.getConf());
    - if (fs.exists(new Path(PIGOUTPUT_DIR)))
    - {
    - fs.delete(new Path(PIGOUTPUT_DIR), true);
    - }
    - } finally {
    - new File(PIG_FILE).delete();
    - }
    - }
    -
    - @Test
        public void testGetInputBytes() throws Exception {
          assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
          File file = new File(TEST_WAREHOUSE_DIR + "/" + SPECIFIC_SIZE_TABLE + "/part-m-00000");
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10671: yarn-cluster mode offers a degraded performance from yarn-client [Spark Branch] (Rui via Xuefu, reviewed by Chengxiang)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f27c6ed
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f27c6ed
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f27c6ed

    Branch: refs/heads/hbase-metastore
    Commit: 3f27c6ed47bbfd8257c70bca0983a7f2e3e74f4a
    Parents: 2df14f9
    Author: xzhang <xzhang@xzdt>
    Authored: Thu May 14 14:20:11 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:03:54 2015 -0700

    ----------------------------------------------------------------------
      .../hive/ql/exec/spark/RemoteHiveSparkClient.java | 10 +++++-----
      .../hadoop/hive/ql/exec/spark/SparkUtilities.java | 11 +++++++----
      .../java/org/apache/hive/spark/client/JobContext.java | 2 +-
      .../org/apache/hive/spark/client/JobContextImpl.java | 7 ++++---
      .../apache/hive/spark/client/SparkClientUtilities.java | 13 ++++++++-----
      5 files changed, 25 insertions(+), 18 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    index 059016d..bae30f3 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
    @@ -28,6 +28,7 @@ import java.net.URISyntaxException;
      import java.util.ArrayList;
      import java.util.List;
      import java.util.Map;
    +import java.util.Set;
      import java.util.concurrent.Future;
      import java.util.concurrent.TimeUnit;

    @@ -167,10 +168,10 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
            try {
              URI fileUri = SparkUtilities.getURI(addedFile);
              if (fileUri != null && !localFiles.contains(fileUri)) {
    + localFiles.add(fileUri);
                if (SparkUtilities.needUploadToHDFS(fileUri, sparkConf)) {
                  fileUri = SparkUtilities.uploadToHDFS(fileUri, hiveConf);
                }
    - localFiles.add(fileUri);
                remoteClient.addFile(fileUri);
              }
            } catch (URISyntaxException e) {
    @@ -184,10 +185,10 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
            try {
              URI jarUri = SparkUtilities.getURI(addedJar);
              if (jarUri != null && !localJars.contains(jarUri)) {
    + localJars.add(jarUri);
                if (SparkUtilities.needUploadToHDFS(jarUri, sparkConf)) {
                  jarUri = SparkUtilities.uploadToHDFS(jarUri, hiveConf);
                }
    - localJars.add(jarUri);
                remoteClient.addJar(jarUri);
              }
            } catch (URISyntaxException e) {
    @@ -224,10 +225,9 @@ public class RemoteHiveSparkClient implements HiveSparkClient {

            // Add jar to current thread class loader dynamically, and add jar paths to JobConf as Spark
            // may need to load classes from this jar in other threads.
    - List<String> addedJars = jc.getAddedJars();
    + Set<String> addedJars = jc.getAddedJars();
            if (addedJars != null && !addedJars.isEmpty()) {
    - SparkClientUtilities.addToClassPath(addedJars.toArray(new String[addedJars.size()]),
    - localJobConf, jc.getLocalTmpDir());
    + SparkClientUtilities.addToClassPath(addedJars, localJobConf, jc.getLocalTmpDir());
              localJobConf.set(Utilities.HIVE_ADDED_JARS, StringUtils.join(addedJars, ";"));
            }


    http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    index 72ab913..c012af8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    @@ -21,6 +21,7 @@ import java.io.File;
      import java.io.IOException;
      import java.net.URI;
      import java.net.URISyntaxException;
    +import java.util.UUID;

      import org.apache.commons.io.FilenameUtils;
      import org.apache.hadoop.fs.FileSystem;
    @@ -76,11 +77,13 @@ public class SparkUtilities {
         * @throws IOException
         */
        public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException {
    - Path tmpDir = SessionState.getHDFSSessionPath(conf);
    + Path localFile = new Path(source.getPath());
    + // give the uploaded file a UUID
    + Path remoteFile = new Path(SessionState.getHDFSSessionPath(conf),
    + UUID.randomUUID() + "-" + getFileName(source));
          FileSystem fileSystem = FileSystem.get(conf);
    - fileSystem.copyFromLocalFile(new Path(source.getPath()), tmpDir);
    - String filePath = tmpDir + File.separator + getFileName(source);
    - Path fullPath = fileSystem.getFileStatus(new Path(filePath)).getPath();
    + fileSystem.copyFromLocalFile(localFile, remoteFile);
    + Path fullPath = fileSystem.getFileStatus(remoteFile).getPath();
          return fullPath.toUri();
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
    index 36e252c..af6332e 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
    @@ -57,7 +57,7 @@ public interface JobContext {
        /**
         * Return all added jar path which added through AddJarJob.
         */
    - List<String> getAddedJars();
    + Set<String> getAddedJars();

        /**
         * Returns a local tmp dir specific to the context

    http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
    index 164d90a..beed8a3 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
    @@ -18,6 +18,7 @@
      package org.apache.hive.spark.client;

      import java.io.File;
    +import java.util.Collections;
      import java.util.List;
      import java.util.Map;
      import java.util.Set;
    @@ -34,14 +35,14 @@ class JobContextImpl implements JobContext {
        private final JavaSparkContext sc;
        private final ThreadLocal<MonitorCallback> monitorCb;
        private final Map<String, List<JavaFutureAction<?>>> monitoredJobs;
    - private final List<String> addedJars;
    + private final Set<String> addedJars;
        private final File localTmpDir;

        public JobContextImpl(JavaSparkContext sc, File localTmpDir) {
          this.sc = sc;
          this.monitorCb = new ThreadLocal<MonitorCallback>();
          monitoredJobs = new ConcurrentHashMap<String, List<JavaFutureAction<?>>>();
    - addedJars = new CopyOnWriteArrayList<String>();
    + addedJars = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
          this.localTmpDir = localTmpDir;
        }

    @@ -64,7 +65,7 @@ class JobContextImpl implements JobContext {
        }

        @Override
    - public List<String> getAddedJars() {
    + public Set<String> getAddedJars() {
          return addedJars;
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
    index 879f8a4..b079ee2 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
    @@ -24,6 +24,7 @@ import java.io.File;
      import java.net.URL;
      import java.net.URLClassLoader;
      import java.util.List;
    +import java.util.Set;

      import org.apache.commons.lang.StringUtils;
      import org.apache.commons.logging.Log;
    @@ -38,9 +39,9 @@ public class SparkClientUtilities {
        /**
         * Add new elements to the classpath.
         *
    - * @param newPaths Array of classpath elements
    + * @param newPaths Set of classpath elements
         */
    - public static void addToClassPath(String[] newPaths, Configuration conf, File localTmpDir)
    + public static void addToClassPath(Set<String> newPaths, Configuration conf, File localTmpDir)
            throws Exception {
          ClassLoader cloader = Thread.currentThread().getContextClassLoader();
          URLClassLoader loader = (URLClassLoader) cloader;
    @@ -74,9 +75,11 @@ public class SparkClientUtilities {
              Path remoteFile = new Path(path);
              Path localFile =
                  new Path(localTmpDir.getAbsolutePath() + File.separator + remoteFile.getName());
    - LOG.info("Copying " + remoteFile + " to " + localFile);
    - FileSystem fs = remoteFile.getFileSystem(conf);
    - fs.copyToLocalFile(remoteFile, localFile);
    + if (!new File(localFile.toString()).exists()) {
    + LOG.info("Copying " + remoteFile + " to " + localFile);
    + FileSystem remoteFS = remoteFile.getFileSystem(conf);
    + remoteFS.copyToLocalFile(remoteFile, localFile);
    + }
              return urlFromPathString(localFile.toString(), conf, localTmpDir);
            } else {
              url = new File(path).toURL();
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10667 : Upgrade Calcite dependency to newest version (Jesus Camacho Rodriguez via Ashutosh Chauhan)

    Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7556361c
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7556361c
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7556361c

    Branch: refs/heads/hbase-metastore
    Commit: 7556361c2a43b47547a9ccdf9f9bec42964a75ae
    Parents: d5fdf14
    Author: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Authored: Tue Jun 2 03:59:00 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Tue Jun 2 13:04:58 2015 -0700

    ----------------------------------------------------------------------
      pom.xml | 2 +-
      .../ql/optimizer/calcite/HiveCalciteUtil.java | 2 +-
      .../ql/optimizer/calcite/HiveRelOptUtil.java | 414 -------------------
      .../calcite/reloperators/HiveProject.java | 5 +-
      .../calcite/reloperators/HiveSemiJoin.java | 79 ++++
      .../calcite/translator/HiveOpConverter.java | 6 +-
      .../hadoop/hive/ql/parse/CalcitePlanner.java | 10 +-
      7 files changed, 91 insertions(+), 427 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/pom.xml
    ----------------------------------------------------------------------
    diff --git a/pom.xml b/pom.xml
    index b21d894..e60bb35 100644
    --- a/pom.xml
    +++ b/pom.xml
    @@ -100,7 +100,7 @@
          <antlr.version>3.4</antlr.version>
          <avro.version>1.7.7</avro.version>
          <bonecp.version>0.8.0.RELEASE</bonecp.version>
    - <calcite.version>1.2.0-incubating</calcite.version>
    + <calcite.version>1.3.0-incubating</calcite.version>
          <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
          <datanucleus-core.version>3.2.10</datanucleus-core.version>
          <datanucleus-rdbms.version>3.2.9</datanucleus-rdbms.version>

    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
    index f4e7c45..199a358 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
    @@ -504,7 +504,7 @@ public class HiveCalciteUtil {
            }

            // 1. Split leaf join predicate to expressions from left, right
    - HiveRelOptUtil.splitJoinCondition(systemFieldList, inputs, pe,
    + RelOptUtil.splitJoinCondition(systemFieldList, inputs, pe,
                joinKeyExprs, filterNulls, null);

            // 2. Collect child projection indexes used

    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
    deleted file mode 100644
    index 2827c3b..0000000
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
    +++ /dev/null
    @@ -1,414 +0,0 @@
    -/**
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -package org.apache.hadoop.hive.ql.optimizer.calcite;
    -
    -import java.util.ArrayList;
    -import java.util.List;
    -
    -import org.apache.calcite.plan.RelOptCluster;
    -import org.apache.calcite.plan.RelOptUtil;
    -import org.apache.calcite.rel.RelNode;
    -import org.apache.calcite.rel.type.RelDataType;
    -import org.apache.calcite.rel.type.RelDataTypeFactory;
    -import org.apache.calcite.rel.type.RelDataTypeField;
    -import org.apache.calcite.rex.RexBuilder;
    -import org.apache.calcite.rex.RexCall;
    -import org.apache.calcite.rex.RexNode;
    -import org.apache.calcite.rex.RexUtil;
    -import org.apache.calcite.sql.SqlKind;
    -import org.apache.calcite.sql.SqlOperator;
    -import org.apache.calcite.sql.fun.SqlStdOperatorTable;
    -import org.apache.calcite.util.ImmutableBitSet;
    -import org.apache.calcite.util.Util;
    -
    -import com.google.common.collect.ImmutableList;
    -
    -
    -public class HiveRelOptUtil extends RelOptUtil {
    -
    - /**
    - * Splits out the equi-join (and optionally, a single non-equi) components
    - * of a join condition, and returns what's left. Projection might be
    - * required by the caller to provide join keys that are not direct field
    - * references.
    - *
    - * @param sysFieldList list of system fields
    - * @param leftRel left join input
    - * @param rightRel right join input
    - * @param condition join condition
    - * @param leftJoinKeys The join keys from the left input which are equi-join
    - * keys
    - * @param rightJoinKeys The join keys from the right input which are
    - * equi-join keys
    - * @param filterNulls The join key positions for which null values will not
    - * match. null values only match for the "is not distinct
    - * from" condition.
    - * @param rangeOp if null, only locate equi-joins; otherwise, locate a
    - * single non-equi join predicate and return its operator
    - * in this list; join keys associated with the non-equi
    - * join predicate are at the end of the key lists
    - * returned
    - * @return What's left, never null
    - */
    - public static RexNode splitJoinCondition(
    - List<RelDataTypeField> sysFieldList,
    - RelNode leftRel,
    - RelNode rightRel,
    - RexNode condition,
    - List<RexNode> leftJoinKeys,
    - List<RexNode> rightJoinKeys,
    - List<Integer> filterNulls,
    - List<SqlOperator> rangeOp) {
    - return splitJoinCondition(
    - sysFieldList,
    - ImmutableList.of(leftRel, rightRel),
    - condition,
    - ImmutableList.of(leftJoinKeys, rightJoinKeys),
    - filterNulls,
    - rangeOp);
    - }
    -
    - /**
    - * Splits out the equi-join (and optionally, a single non-equi) components
    - * of a join condition, and returns what's left. Projection might be
    - * required by the caller to provide join keys that are not direct field
    - * references.
    - *
    - * @param sysFieldList list of system fields
    - * @param inputs join inputs
    - * @param condition join condition
    - * @param joinKeys The join keys from the inputs which are equi-join
    - * keys
    - * @param filterNulls The join key positions for which null values will not
    - * match. null values only match for the "is not distinct
    - * from" condition.
    - * @param rangeOp if null, only locate equi-joins; otherwise, locate a
    - * single non-equi join predicate and return its operator
    - * in this list; join keys associated with the non-equi
    - * join predicate are at the end of the key lists
    - * returned
    - * @return What's left, never null
    - */
    - public static RexNode splitJoinCondition(
    - List<RelDataTypeField> sysFieldList,
    - List<RelNode> inputs,
    - RexNode condition,
    - List<List<RexNode>> joinKeys,
    - List<Integer> filterNulls,
    - List<SqlOperator> rangeOp) {
    - final List<RexNode> nonEquiList = new ArrayList<>();
    -
    - splitJoinCondition(
    - sysFieldList,
    - inputs,
    - condition,
    - joinKeys,
    - filterNulls,
    - rangeOp,
    - nonEquiList);
    -
    - // Convert the remainders into a list that are AND'ed together.
    - return RexUtil.composeConjunction(
    - inputs.get(0).getCluster().getRexBuilder(), nonEquiList, false);
    - }
    -
    - private static void splitJoinCondition(
    - List<RelDataTypeField> sysFieldList,
    - List<RelNode> inputs,
    - RexNode condition,
    - List<List<RexNode>> joinKeys,
    - List<Integer> filterNulls,
    - List<SqlOperator> rangeOp,
    - List<RexNode> nonEquiList) {
    - final int sysFieldCount = sysFieldList.size();
    - final RelOptCluster cluster = inputs.get(0).getCluster();
    - final RexBuilder rexBuilder = cluster.getRexBuilder();
    - final RelDataTypeFactory typeFactory = cluster.getTypeFactory();
    -
    - int[] firstFieldInputs = new int[inputs.size()];
    - int totalFieldCount = 0;
    - for (int i = 0; i < inputs.size(); i++) {
    - firstFieldInputs[i] = totalFieldCount + sysFieldCount;
    - totalFieldCount += sysFieldCount
    - + inputs.get(i).getRowType().getFieldCount();
    - }
    -
    - // adjustment array
    - int[] adjustments = new int[totalFieldCount];
    - for (int i = 0; i < inputs.size(); i++) {
    - int limit = i == inputs.size() - 1
    - ? totalFieldCount : firstFieldInputs[i + 1];
    - for (int j = firstFieldInputs[i]; j < limit; j++) {
    - adjustments[j] = -firstFieldInputs[i];
    - }
    - }
    -
    - if (condition instanceof RexCall) {
    - RexCall call = (RexCall) condition;
    - if (call.getOperator() == SqlStdOperatorTable.AND) {
    - for (RexNode operand : call.getOperands()) {
    - splitJoinCondition(
    - sysFieldList,
    - inputs,
    - operand,
    - joinKeys,
    - filterNulls,
    - rangeOp,
    - nonEquiList);
    - }
    - return;
    - }
    -
    - RexNode leftKey = null;
    - RexNode rightKey = null;
    - int leftInput = 0;
    - int rightInput = 0;
    - List<RelDataTypeField> leftFields = null;
    - List<RelDataTypeField> rightFields = null;
    - boolean reverse = false;
    -
    - SqlKind kind = call.getKind();
    -
    - // Only consider range operators if we haven't already seen one
    - if ((kind == SqlKind.EQUALS)
    - || (filterNulls != null
    - && kind == SqlKind.IS_NOT_DISTINCT_FROM)
    - || (rangeOp != null
    - && rangeOp.isEmpty()
    - && (kind == SqlKind.GREATER_THAN
    - || kind == SqlKind.GREATER_THAN_OR_EQUAL
    - || kind == SqlKind.LESS_THAN
    - || kind == SqlKind.LESS_THAN_OR_EQUAL))) {
    - final List<RexNode> operands = call.getOperands();
    - RexNode op0 = operands.get(0);
    - RexNode op1 = operands.get(1);
    -
    - final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
    - final ImmutableBitSet projRefs1 = InputFinder.bits(op1);
    -
    - boolean foundBothInputs = false;
    - for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
    - final int lowerLimit = firstFieldInputs[i];
    - final int upperLimit = i == inputs.size() - 1
    - ? totalFieldCount : firstFieldInputs[i + 1];
    - if (projRefs0.nextSetBit(lowerLimit) != -1
    - && projRefs0.nextSetBit(upperLimit) == -1
    - && projRefs0.nextSetBit(0) == projRefs0.nextSetBit(lowerLimit)
    - && projRefs0.nextSetBit(lowerLimit) < upperLimit) {
    - if (leftKey == null) {
    - leftKey = op0;
    - leftInput = i;
    - leftFields = inputs.get(leftInput).getRowType().getFieldList();
    - } else {
    - rightKey = op0;
    - rightInput = i;
    - rightFields = inputs.get(rightInput).getRowType().getFieldList();
    - reverse = true;
    - foundBothInputs = true;
    - }
    - } else if (projRefs1.nextSetBit(lowerLimit) != -1
    - && projRefs1.nextSetBit(upperLimit) == -1
    - && projRefs1.nextSetBit(0) == projRefs1.nextSetBit(lowerLimit)
    - && projRefs1.nextSetBit(lowerLimit) < upperLimit) {
    - if (leftKey == null) {
    - leftKey = op1;
    - leftInput = i;
    - leftFields = inputs.get(leftInput).getRowType().getFieldList();
    - } else {
    - rightKey = op1;
    - rightInput = i;
    - rightFields = inputs.get(rightInput).getRowType().getFieldList();
    - foundBothInputs = true;
    - }
    - }
    - }
    -
    - if ((leftKey != null) && (rightKey != null)) {
    - // replace right Key input ref
    - rightKey =
    - rightKey.accept(
    - new RelOptUtil.RexInputConverter(
    - rexBuilder,
    - rightFields,
    - rightFields,
    - adjustments));
    -
    - // left key only needs to be adjusted if there are system
    - // fields, but do it for uniformity
    - leftKey =
    - leftKey.accept(
    - new RelOptUtil.RexInputConverter(
    - rexBuilder,
    - leftFields,
    - leftFields,
    - adjustments));
    -
    - RelDataType leftKeyType = leftKey.getType();
    - RelDataType rightKeyType = rightKey.getType();
    -
    - if (leftKeyType != rightKeyType) {
    - // perform casting
    - RelDataType targetKeyType =
    - typeFactory.leastRestrictive(
    - ImmutableList.of(leftKeyType, rightKeyType));
    -
    - if (targetKeyType == null) {
    - throw Util.newInternal(
    - "Cannot find common type for join keys "
    - + leftKey + " (type " + leftKeyType + ") and "
    - + rightKey + " (type " + rightKeyType + ")");
    - }
    -
    - if (leftKeyType != targetKeyType) {
    - leftKey =
    - rexBuilder.makeCast(targetKeyType, leftKey);
    - }
    -
    - if (rightKeyType != targetKeyType) {
    - rightKey =
    - rexBuilder.makeCast(targetKeyType, rightKey);
    - }
    - }
    - }
    - }
    -
    - if ((rangeOp == null)
    - && ((leftKey == null) || (rightKey == null))) {
    - // no equality join keys found yet:
    - // try transforming the condition to
    - // equality "join" conditions, e.g.
    - // f(LHS) > 0 ===> ( f(LHS) > 0 ) = TRUE,
    - // and make the RHS produce TRUE, but only if we're strictly
    - // looking for equi-joins
    - final ImmutableBitSet projRefs = InputFinder.bits(condition);
    - leftKey = null;
    - rightKey = null;
    -
    - boolean foundInput = false;
    - for (int i = 0; i < inputs.size() && !foundInput; i++) {
    - final int lowerLimit = firstFieldInputs[i];
    - final int upperLimit = i == inputs.size() - 1
    - ? totalFieldCount : firstFieldInputs[i + 1];
    - if (projRefs.nextSetBit(lowerLimit) < upperLimit) {
    - leftInput = i;
    - leftFields = inputs.get(leftInput).getRowType().getFieldList();
    -
    - leftKey = condition.accept(
    - new RelOptUtil.RexInputConverter(
    - rexBuilder,
    - leftFields,
    - leftFields,
    - adjustments));
    -
    - rightKey = rexBuilder.makeLiteral(true);
    -
    - // effectively performing an equality comparison
    - kind = SqlKind.EQUALS;
    -
    - foundInput = true;
    - }
    - }
    - }
    -
    - if ((leftKey != null) && (rightKey != null)) {
    - // found suitable join keys
    - // add them to key list, ensuring that if there is a
    - // non-equi join predicate, it appears at the end of the
    - // key list; also mark the null filtering property
    - addJoinKey(
    - joinKeys.get(leftInput),
    - leftKey,
    - (rangeOp != null) && !rangeOp.isEmpty());
    - addJoinKey(
    - joinKeys.get(rightInput),
    - rightKey,
    - (rangeOp != null) && !rangeOp.isEmpty());
    - if (filterNulls != null
    - && kind == SqlKind.EQUALS) {
    - // nulls are considered not matching for equality comparison
    - // add the position of the most recently inserted key
    - filterNulls.add(joinKeys.get(leftInput).size() - 1);
    - }
    - if (rangeOp != null
    - && kind != SqlKind.EQUALS
    - && kind != SqlKind.IS_DISTINCT_FROM) {
    - if (reverse) {
    - kind = reverse(kind);
    - }
    - rangeOp.add(op(kind, call.getOperator()));
    - }
    - return;
    - } // else fall through and add this condition as nonEqui condition
    - }
    -
    - // The operator is not of RexCall type
    - // So we fail. Fall through.
    - // Add this condition to the list of non-equi-join conditions.
    - nonEquiList.add(condition);
    - }
    -
    - private static SqlKind reverse(SqlKind kind) {
    - switch (kind) {
    - case GREATER_THAN:
    - return SqlKind.LESS_THAN;
    - case GREATER_THAN_OR_EQUAL:
    - return SqlKind.LESS_THAN_OR_EQUAL;
    - case LESS_THAN:
    - return SqlKind.GREATER_THAN;
    - case LESS_THAN_OR_EQUAL:
    - return SqlKind.GREATER_THAN_OR_EQUAL;
    - default:
    - return kind;
    - }
    - }
    -
    - private static SqlOperator op(SqlKind kind, SqlOperator operator) {
    - switch (kind) {
    - case EQUALS:
    - return SqlStdOperatorTable.EQUALS;
    - case NOT_EQUALS:
    - return SqlStdOperatorTable.NOT_EQUALS;
    - case GREATER_THAN:
    - return SqlStdOperatorTable.GREATER_THAN;
    - case GREATER_THAN_OR_EQUAL:
    - return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL;
    - case LESS_THAN:
    - return SqlStdOperatorTable.LESS_THAN;
    - case LESS_THAN_OR_EQUAL:
    - return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
    - case IS_DISTINCT_FROM:
    - return SqlStdOperatorTable.IS_DISTINCT_FROM;
    - case IS_NOT_DISTINCT_FROM:
    - return SqlStdOperatorTable.IS_NOT_DISTINCT_FROM;
    - default:
    - return operator;
    - }
    - }
    -
    - private static void addJoinKey(
    - List<RexNode> joinKeyList,
    - RexNode key,
    - boolean preserveLastElementInList) {
    - if (!joinKeyList.isEmpty() && preserveLastElementInList) {
    - joinKeyList.add(joinKeyList.size() - 1, key);
    - } else {
    - joinKeyList.add(key);
    - }
    - }
    -
    -}

    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
    index cf0c462..4b7887a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
    @@ -39,10 +39,10 @@ import org.apache.calcite.util.Util;
      import org.apache.calcite.util.mapping.Mapping;
      import org.apache.calcite.util.mapping.MappingType;
      import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
    +import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
      import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
      import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
    -import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
    -import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCost;
    +
      import com.google.common.collect.ImmutableList;

      public class HiveProject extends Project implements HiveRelNode {
    @@ -68,6 +68,7 @@ public class HiveProject extends Project implements HiveRelNode {
        public HiveProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode child,
            List<? extends RexNode> exps, RelDataType rowType, int flags) {
          super(cluster, traitSet, child, exps, rowType, flags);
    + assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
          virtualCols = ImmutableList.copyOf(HiveCalciteUtil.getVirtualCols(exps));
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
    new file mode 100644
    index 0000000..dd1691c
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
    @@ -0,0 +1,79 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
    +
    +import org.apache.calcite.plan.RelOptCluster;
    +import org.apache.calcite.plan.RelOptCost;
    +import org.apache.calcite.plan.RelOptPlanner;
    +import org.apache.calcite.plan.RelTraitSet;
    +import org.apache.calcite.rel.RelNode;
    +import org.apache.calcite.rel.core.JoinInfo;
    +import org.apache.calcite.rel.core.JoinRelType;
    +import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory;
    +import org.apache.calcite.rel.core.SemiJoin;
    +import org.apache.calcite.rel.metadata.RelMetadataQuery;
    +import org.apache.calcite.rex.RexNode;
    +import org.apache.calcite.util.ImmutableIntList;
    +
    +public class HiveSemiJoin extends SemiJoin implements HiveRelNode {
    +
    + public static final SemiJoinFactory HIVE_SEMIJOIN_FACTORY = new HiveSemiJoinFactoryImpl();
    +
    + public HiveSemiJoin(RelOptCluster cluster,
    + RelTraitSet traitSet,
    + RelNode left,
    + RelNode right,
    + RexNode condition,
    + ImmutableIntList leftKeys,
    + ImmutableIntList rightKeys) {
    + super(cluster, traitSet, left, right, condition, leftKeys, rightKeys);
    + }
    +
    + @Override
    + public SemiJoin copy(RelTraitSet traitSet, RexNode condition,
    + RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) {
    + final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
    + return new HiveSemiJoin(getCluster(), traitSet, left, right, condition,
    + joinInfo.leftKeys, joinInfo.rightKeys);
    + }
    +
    + @Override
    + public void implement(Implementor implementor) {
    + }
    +
    + @Override
    + public RelOptCost computeSelfCost(RelOptPlanner planner) {
    + return RelMetadataQuery.getNonCumulativeCost(this);
    + }
    +
    + /**
    + * Implementation of {@link SemiJoinFactory} that returns
    + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin}
    + * .
    + */
    + private static class HiveSemiJoinFactoryImpl implements SemiJoinFactory {
    + @Override
    + public RelNode createSemiJoin(RelNode left, RelNode right,
    + RexNode condition) {
    + final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
    + final RelOptCluster cluster = left.getCluster();
    + return new HiveSemiJoin(cluster, left.getTraitSet(), left, right, condition,
    + joinInfo.leftKeys, joinInfo.rightKeys);
    + }
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
    index 4f19caf..9c21238 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
    @@ -32,7 +32,6 @@ import org.apache.calcite.rel.RelDistribution;
      import org.apache.calcite.rel.RelDistribution.Type;
      import org.apache.calcite.rel.RelFieldCollation;
      import org.apache.calcite.rel.RelNode;
    -import org.apache.calcite.rel.core.SemiJoin;
      import org.apache.calcite.rel.rules.MultiJoin;
      import org.apache.calcite.rex.RexInputRef;
      import org.apache.calcite.rex.RexLiteral;
    @@ -63,6 +62,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
    +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
    @@ -156,8 +156,8 @@ public class HiveOpConverter {
            return visit((MultiJoin) rn);
          } else if (rn instanceof HiveJoin) {
            return visit((HiveJoin) rn);
    - } else if (rn instanceof SemiJoin) {
    - SemiJoin sj = (SemiJoin) rn;
    + } else if (rn instanceof HiveSemiJoin) {
    + HiveSemiJoin sj = (HiveSemiJoin) rn;
            HiveJoin hj = HiveJoin.getJoin(sj.getCluster(), sj.getLeft(), sj.getRight(),
                sj.getCondition(), sj.getJoinType(), true);
            return visit(hj);

    http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
    index d545bf0..02fc796 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
    @@ -59,8 +59,6 @@ import org.apache.calcite.rel.core.Filter;
      import org.apache.calcite.rel.core.Join;
      import org.apache.calcite.rel.core.JoinRelType;
      import org.apache.calcite.rel.core.Project;
    -import org.apache.calcite.rel.core.RelFactories;
    -import org.apache.calcite.rel.core.SemiJoin;
      import org.apache.calcite.rel.core.Sort;
      import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
      import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
    @@ -121,7 +119,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.Unsu
      import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
      import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfigContext;
      import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
    -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
      import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
      import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
      import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
    @@ -133,6 +130,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
    +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
      import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
    @@ -956,7 +954,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
            // 5. Projection Pruning
            HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
                HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
    - RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
    + HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
                HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
            basePlan = fieldTrimmer.trim(basePlan);

    @@ -1196,7 +1194,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
              List<RexNode> leftJoinKeys = new ArrayList<RexNode>();
              List<RexNode> rightJoinKeys = new ArrayList<RexNode>();

    - RexNode nonEquiConds = HiveRelOptUtil.splitJoinCondition(sysFieldList, leftRel, rightRel,
    + RexNode nonEquiConds = RelOptUtil.splitJoinCondition(sysFieldList, leftRel, rightRel,
                  calciteJoinCond, leftJoinKeys, rightJoinKeys, null, null);

              if (!nonEquiConds.isAlwaysTrue()) {
    @@ -1211,7 +1209,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
                  HiveProject.DEFAULT_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0,
                  leftKeys, rightKeys);

    - joinRel = new SemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0],
    + joinRel = new HiveSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0],
                  inputRels[1], calciteJoinCond, ImmutableIntList.copyOf(leftKeys),
                  ImmutableIntList.copyOf(rightKeys));
            } else {
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10864 : CBO (Calcite Return Path): auto_join2.q returning wrong results (Jesus Camacho Rodriguez via Ashutosh Chauhan)

    Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a9eaa204
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a9eaa204
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a9eaa204

    Branch: refs/heads/hbase-metastore
    Commit: a9eaa2042eff0e4bbf27ba9ed3afc58fff43e89e
    Parents: 2b8d547
    Author: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Authored: Thu May 28 11:26:00 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Jun 1 11:40:43 2015 -0700

    ----------------------------------------------------------------------
      .../ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/a9eaa204/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
    index 30db8fd..11c3d23 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
    @@ -98,11 +98,11 @@ public class HiveInsertExchange4JoinRule extends RelOptRule {
            for (int j = 0; j < joinPredInfo.getEquiJoinPredicateElements().size(); j++) {
              JoinLeafPredicateInfo joinLeafPredInfo = joinPredInfo.
                  getEquiJoinPredicateElements().get(j);
    + keyListBuilder.add(joinLeafPredInfo.getJoinKeyExprs(i).get(0));
              for (int pos : joinLeafPredInfo.getProjsJoinKeysInChildSchema(i)) {
                if (!joinKeyPositions.contains(pos)) {
                  joinKeyPositions.add(pos);
                  collationListBuilder.add(new RelFieldCollation(pos));
    - keyListBuilder.add(joinLeafPredInfo.getJoinKeyExprs(i).get(0));
                }
              }
            }
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10878 : Add tests to cover avg() function for 'x preceding and y preceding' windowing spec (Aihua Xu via Ashutosh Chauhan)

    Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/715871a0
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/715871a0
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/715871a0

    Branch: refs/heads/hbase-metastore
    Commit: 715871a0e66c9686a707986109316dba358b5c88
    Parents: e22aaf6
    Author: Aihua Xu <aihuaxu@gmail.com>
    Authored: Mon Jun 1 07:56:00 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Tue Jun 2 09:55:56 2015 -0700

    ----------------------------------------------------------------------
      .../clientpositive/windowing_windowspec2.q | 6 +
      .../clientpositive/windowing_windowspec2.q.out | 434 +++++++++++++++++++
      2 files changed, 440 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/715871a0/ql/src/test/queries/clientpositive/windowing_windowspec2.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec2.q b/ql/src/test/queries/clientpositive/windowing_windowspec2.q
    index e270a86..3c5bc3d 100644
    --- a/ql/src/test/queries/clientpositive/windowing_windowspec2.q
    +++ b/ql/src/test/queries/clientpositive/windowing_windowspec2.q
    @@ -23,6 +23,12 @@ select ts, f, sum(f) over (partition by ts order by f rows between unbounded pre
      select ts, f, sum(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100;
      select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100;

    +-- avg
    +select ts, f, avg(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100;
    +select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100;
    +select ts, f, avg(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100;
    +select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100;
    +
      -- count
      select ts, f, count(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100;
      select ts, f, count(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100;

    http://git-wip-us.apache.org/repos/asf/hive/blob/715871a0/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/windowing_windowspec2.q.out b/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
    index a318398..e71a03f 100644
    --- a/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
    +++ b/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
    @@ -478,6 +478,440 @@ POSTHOOK: Input: default@over10k
      2013-03-01 09:11:58.703072 71.68 802.1099938452244
      2013-03-01 09:11:58.703072 79.46 882.1299904882908
      2013-03-01 09:11:58.703072 80.02 963.4199914038181
    +PREHOOK: query: -- avg
    +select ts, f, avg(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- avg
    +select ts, f, avg(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +2013-03-01 09:11:58.70307 3.17 NULL
    +2013-03-01 09:11:58.70307 10.89 3.1700000762939453
    +2013-03-01 09:11:58.70307 14.54 7.03000020980835
    +2013-03-01 09:11:58.70307 14.78 12.71500015258789
    +2013-03-01 09:11:58.70307 17.85 14.65999984741211
    +2013-03-01 09:11:58.70307 20.61 16.31500005722046
    +2013-03-01 09:11:58.70307 28.69 19.230000495910645
    +2013-03-01 09:11:58.70307 29.22 24.65000057220459
    +2013-03-01 09:11:58.70307 31.17 28.954999923706055
    +2013-03-01 09:11:58.70307 38.35 30.19499969482422
    +2013-03-01 09:11:58.70307 38.61 34.75999927520752
    +2013-03-01 09:11:58.70307 39.48 38.47999954223633
    +2013-03-01 09:11:58.70307 40.54 39.045000076293945
    +2013-03-01 09:11:58.70307 41.6 40.010000228881836
    +2013-03-01 09:11:58.70307 46.08 41.06999969482422
    +2013-03-01 09:11:58.70307 54.36 43.84000015258789
    +2013-03-01 09:11:58.70307 56.94 50.220001220703125
    +2013-03-01 09:11:58.70307 64.96 55.64999961853027
    +2013-03-01 09:11:58.70307 73.52 60.94999885559082
    +2013-03-01 09:11:58.70307 78.58 69.23999786376953
    +2013-03-01 09:11:58.70307 81.41 76.04999923706055
    +2013-03-01 09:11:58.70307 84.71 79.99500274658203
    +2013-03-01 09:11:58.70307 87.43 83.06000137329102
    +2013-03-01 09:11:58.70307 91.36 86.06999969482422
    +2013-03-01 09:11:58.70307 92.96 89.39500045776367
    +2013-03-01 09:11:58.70307 95.04 92.15999984741211
    +2013-03-01 09:11:58.703071 0.83 NULL
    +2013-03-01 09:11:58.703071 1.99 0.8299999833106995
    +2013-03-01 09:11:58.703071 3.73 1.4099999964237213
    +2013-03-01 09:11:58.703071 8.86 2.8600000143051147
    +2013-03-01 09:11:58.703071 10.62 6.294999837875366
    +2013-03-01 09:11:58.703071 11.32 9.739999771118164
    +2013-03-01 09:11:58.703071 12.83 10.96999979019165
    +2013-03-01 09:11:58.703071 14.7 12.074999809265137
    +2013-03-01 09:11:58.703071 14.96 13.764999866485596
    +2013-03-01 09:11:58.703071 17.58 14.829999923706055
    +2013-03-01 09:11:58.703071 19.1 16.269999980926514
    +2013-03-01 09:11:58.703071 21.01 18.34000015258789
    +2013-03-01 09:11:58.703071 26.95 20.05500030517578
    +2013-03-01 09:11:58.703071 27.23 23.980000495910645
    +2013-03-01 09:11:58.703071 29.07 27.09000015258789
    +2013-03-01 09:11:58.703071 29.71 28.149999618530273
    +2013-03-01 09:11:58.703071 31.84 29.389999389648438
    +2013-03-01 09:11:58.703071 31.94 30.774999618530273
    +2013-03-01 09:11:58.703071 35.32 31.890000343322754
    +2013-03-01 09:11:58.703071 37.32 33.63000011444092
    +2013-03-01 09:11:58.703071 38.5 36.31999969482422
    +2013-03-01 09:11:58.703071 42.08 37.90999984741211
    +2013-03-01 09:11:58.703071 44.3 40.290000915527344
    +2013-03-01 09:11:58.703071 44.66 43.19000053405762
    +2013-03-01 09:11:58.703071 46.84 44.47999954223633
    +2013-03-01 09:11:58.703071 48.89 45.75
    +2013-03-01 09:11:58.703071 49.64 47.864999771118164
    +2013-03-01 09:11:58.703071 50.28 49.26499938964844
    +2013-03-01 09:11:58.703071 52.09 49.959999084472656
    +2013-03-01 09:11:58.703071 53.26 51.18499946594238
    +2013-03-01 09:11:58.703071 54.09 52.67499923706055
    +2013-03-01 09:11:58.703071 56.45 53.67499923706055
    +2013-03-01 09:11:58.703071 56.76 55.27000045776367
    +2013-03-01 09:11:58.703071 61.41 56.60499954223633
    +2013-03-01 09:11:58.703071 61.88 59.084999084472656
    +2013-03-01 09:11:58.703071 63.03 61.64500045776367
    +2013-03-01 09:11:58.703071 64.55 62.454999923706055
    +2013-03-01 09:11:58.703071 68.62 63.790000915527344
    +2013-03-01 09:11:58.703071 76.13 66.58500289916992
    +2013-03-01 09:11:58.703071 79.05 72.375
    +2013-03-01 09:11:58.703071 80.43 77.59000015258789
    +2013-03-01 09:11:58.703071 81.41 79.7400016784668
    +2013-03-01 09:11:58.703071 82.85 80.92000198364258
    +2013-03-01 09:11:58.703071 83.98 82.13000106811523
    +2013-03-01 09:11:58.703071 84.21 83.41500091552734
    +2013-03-01 09:11:58.703071 85.55 84.09500122070312
    +2013-03-01 09:11:58.703071 87.93 84.88000106811523
    +2013-03-01 09:11:58.703071 88.93 86.7400016784668
    +2013-03-01 09:11:58.703071 94.27 88.43000030517578
    +2013-03-01 09:11:58.703071 99.45 91.5999984741211
    +2013-03-01 09:11:58.703072 0.36 NULL
    +2013-03-01 09:11:58.703072 0.48 0.36000001430511475
    +2013-03-01 09:11:58.703072 0.79 0.42000000178813934
    +2013-03-01 09:11:58.703072 1.27 0.635000005364418
    +2013-03-01 09:11:58.703072 4.48 1.030000001192093
    +2013-03-01 09:11:58.703072 9.0 2.875
    +2013-03-01 09:11:58.703072 23.27 6.740000009536743
    +2013-03-01 09:11:58.703072 25.13 16.135000228881836
    +2013-03-01 09:11:58.703072 25.34 24.199999809265137
    +2013-03-01 09:11:58.703072 25.91 25.234999656677246
    +2013-03-01 09:11:58.703072 29.01 25.625
    +2013-03-01 09:11:58.703072 30.47 27.460000038146973
    +2013-03-01 09:11:58.703072 37.95 29.739999771118164
    +2013-03-01 09:11:58.703072 39.3 34.21000003814697
    +2013-03-01 09:11:58.703072 45.91 38.625
    +2013-03-01 09:11:58.703072 52.44 42.60499954223633
    +2013-03-01 09:11:58.703072 54.1 49.17499923706055
    +2013-03-01 09:11:58.703072 56.7 53.26999855041504
    +2013-03-01 09:11:58.703072 58.77 55.39999961853027
    +2013-03-01 09:11:58.703072 62.09 57.73500061035156
    +2013-03-01 09:11:58.703072 68.2 60.43000030517578
    +2013-03-01 09:11:58.703072 71.68 65.14499855041504
    +2013-03-01 09:11:58.703072 79.46 69.93999862670898
    +2013-03-01 09:11:58.703072 80.02 75.56999969482422
    +PREHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +POSTHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +2013-03-01 09:11:58.70307 3.17 NULL
    +2013-03-01 09:11:58.70307 10.89 3.1700000762939453
    +2013-03-01 09:11:58.70307 14.54 7.03000020980835
    +2013-03-01 09:11:58.70307 14.78 9.533333460489908
    +2013-03-01 09:11:58.70307 17.85 10.84500002861023
    +2013-03-01 09:11:58.70307 20.61 12.24600009918213
    +2013-03-01 09:11:58.70307 28.69 13.640000184377035
    +2013-03-01 09:11:58.70307 29.22 15.790000234331403
    +2013-03-01 09:11:58.70307 31.17 17.46875011920929
    +2013-03-01 09:11:58.70307 38.35 18.99111122555203
    +2013-03-01 09:11:58.70307 38.61 20.926999950408934
    +2013-03-01 09:11:58.70307 39.48 22.534545464949176
    +2013-03-01 09:11:58.70307 40.54 23.94666663805644
    +2013-03-01 09:11:58.70307 41.6 25.22307696709266
    +2013-03-01 09:11:58.70307 46.08 26.39285707473755
    +2013-03-01 09:11:58.70307 54.36 27.70533339182536
    +2013-03-01 09:11:58.70307 56.94 29.371250092983246
    +2013-03-01 09:11:58.70307 64.96 30.992941183202408
    +2013-03-01 09:11:58.70307 73.52 32.8799999554952
    +2013-03-01 09:11:58.70307 78.58 35.01894714957789
    +2013-03-01 09:11:58.70307 81.41 37.196999883651735
    +2013-03-01 09:11:58.70307 84.71 39.30238101595924
    +2013-03-01 09:11:58.70307 87.43 41.36636365543712
    +2013-03-01 09:11:58.70307 91.36 43.369130466295324
    +2013-03-01 09:11:58.70307 92.96 45.368750055631004
    +2013-03-01 09:11:58.70307 95.04 47.272400016784665
    +2013-03-01 09:11:58.703071 0.83 NULL
    +2013-03-01 09:11:58.703071 1.99 0.8299999833106995
    +2013-03-01 09:11:58.703071 3.73 1.4099999964237213
    +2013-03-01 09:11:58.703071 8.86 2.1833333373069763
    +2013-03-01 09:11:58.703071 10.62 3.8524999171495438
    +2013-03-01 09:11:58.703071 11.32 5.205999910831451
    +2013-03-01 09:11:58.703071 12.83 6.224999874830246
    +2013-03-01 09:11:58.703071 14.7 7.168571310383933
    +2013-03-01 09:11:58.703071 14.96 8.109999872744083
    +2013-03-01 09:11:58.703071 17.58 8.871111002233294
    +2013-03-01 09:11:58.703071 19.1 9.74199989438057
    +2013-03-01 09:11:58.703071 21.01 10.592727211388675
    +2013-03-01 09:11:58.703071 26.95 11.460833296179771
    +2013-03-01 09:11:58.703071 27.23 12.652307716699747
    +2013-03-01 09:11:58.703071 29.07 13.693571418523788
    +2013-03-01 09:11:58.703071 29.71 14.718666636943817
    +2013-03-01 09:11:58.703071 31.84 15.65562491491437
    +2013-03-01 09:11:58.703071 31.94 16.607646987718695
    +2013-03-01 09:11:58.703071 35.32 17.459444406959747
    +2013-03-01 09:11:58.703071 37.32 18.399473632636823
    +2013-03-01 09:11:58.703071 38.5 19.345499935746194
    +2013-03-01 09:11:58.703071 42.08 20.257618986424944
    +2013-03-01 09:11:58.703071 44.3 21.249545479362663
    +2013-03-01 09:11:58.703071 44.66 22.251739121001698
    +2013-03-01 09:11:58.703071 46.84 23.1854166512688
    +2013-03-01 09:11:58.703071 48.89 24.131599991321565
    +2013-03-01 09:11:58.703071 49.64 25.083846122026443
    +2013-03-01 09:11:58.703071 50.28 25.993333280086517
    +2013-03-01 09:11:58.703071 52.09 26.8607141907726
    +2013-03-01 09:11:58.703071 53.26 27.73068956876623
    +2013-03-01 09:11:58.703071 54.09 28.581666527191796
    +2013-03-01 09:11:58.703071 56.45 29.40451599897877
    +2013-03-01 09:11:58.703071 56.76 30.24968739785254
    +2013-03-01 09:11:58.703071 61.41 31.05303015311559
    +2013-03-01 09:11:58.703071 61.88 31.945882202947843
    +2013-03-01 09:11:58.703071 63.03 32.801142741952624
    +2013-03-01 09:11:58.703071 64.55 33.64083318743441
    +2013-03-01 09:11:58.703071 68.62 34.47621615674045
    +2013-03-01 09:11:58.703071 76.13 35.37473685647312
    +2013-03-01 09:11:58.703071 79.05 36.419743533317856
    +2013-03-01 09:11:58.703071 80.43 37.48550002127886
    +2013-03-01 09:11:58.703071 81.41 38.532926857471466
    +2013-03-01 09:11:58.703071 82.85 39.553809638534275
    +2013-03-01 09:11:58.703071 83.98 40.56069775098978
    +2013-03-01 09:11:58.703071 84.21 41.547500151124865
    +2013-03-01 09:11:58.703071 85.55 42.49555568297704
    +2013-03-01 09:11:58.703071 87.93 43.43152193012445
    +2013-03-01 09:11:58.703071 88.93 44.378298065763836
    +2013-03-01 09:11:58.703071 94.27 45.30645852908492
    +2013-03-01 09:11:58.703071 99.45 46.305714408962096
    +2013-03-01 09:11:58.703072 0.36 NULL
    +2013-03-01 09:11:58.703072 0.48 0.36000001430511475
    +2013-03-01 09:11:58.703072 0.79 0.42000000178813934
    +2013-03-01 09:11:58.703072 1.27 0.5433333416779836
    +2013-03-01 09:11:58.703072 4.48 0.7250000014901161
    +2013-03-01 09:11:58.703072 9.0 1.47600000500679
    +2013-03-01 09:11:58.703072 23.27 2.730000004172325
    +2013-03-01 09:11:58.703072 25.13 5.664285783256803
    +2013-03-01 09:11:58.703072 25.34 8.097499955445528
    +2013-03-01 09:11:58.703072 25.91 10.013333310683569
    +2013-03-01 09:11:58.703072 29.01 11.602999964356423
    +2013-03-01 09:11:58.703072 30.47 13.185454533858733
    +2013-03-01 09:11:58.703072 37.95 14.62583326548338
    +2013-03-01 09:11:58.703072 39.3 16.419999996056923
    +2013-03-01 09:11:58.703072 45.91 18.05428565612861
    +2013-03-01 09:11:58.703072 52.44 19.911333268880846
    +2013-03-01 09:11:58.703072 54.1 21.944374853745103
    +2013-03-01 09:11:58.703072 56.7 23.835882125531924
    +2013-03-01 09:11:58.703072 58.77 25.661666494276787
    +2013-03-01 09:11:58.703072 62.09 27.404210387091887
    +2013-03-01 09:11:58.703072 68.2 29.138499875366687
    +2013-03-01 09:11:58.703072 71.68 30.998571164551237
    +2013-03-01 09:11:58.703072 79.46 32.84772703457963
    +2013-03-01 09:11:58.703072 80.02 34.87434755848802
    +PREHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +POSTHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +2013-03-01 09:11:58.70307 3.17 12.71500015258789
    +2013-03-01 09:11:58.70307 10.89 14.65999984741211
    +2013-03-01 09:11:58.70307 14.54 16.31500005722046
    +2013-03-01 09:11:58.70307 14.78 19.230000495910645
    +2013-03-01 09:11:58.70307 17.85 24.65000057220459
    +2013-03-01 09:11:58.70307 20.61 28.954999923706055
    +2013-03-01 09:11:58.70307 28.69 30.19499969482422
    +2013-03-01 09:11:58.70307 29.22 34.75999927520752
    +2013-03-01 09:11:58.70307 31.17 38.47999954223633
    +2013-03-01 09:11:58.70307 38.35 39.045000076293945
    +2013-03-01 09:11:58.70307 38.61 40.010000228881836
    +2013-03-01 09:11:58.70307 39.48 41.06999969482422
    +2013-03-01 09:11:58.70307 40.54 43.84000015258789
    +2013-03-01 09:11:58.70307 41.6 50.220001220703125
    +2013-03-01 09:11:58.70307 46.08 55.64999961853027
    +2013-03-01 09:11:58.70307 54.36 60.94999885559082
    +2013-03-01 09:11:58.70307 56.94 69.23999786376953
    +2013-03-01 09:11:58.70307 64.96 76.04999923706055
    +2013-03-01 09:11:58.70307 73.52 79.99500274658203
    +2013-03-01 09:11:58.70307 78.58 83.06000137329102
    +2013-03-01 09:11:58.70307 81.41 86.06999969482422
    +2013-03-01 09:11:58.70307 84.71 89.39500045776367
    +2013-03-01 09:11:58.70307 87.43 92.15999984741211
    +2013-03-01 09:11:58.70307 91.36 94.0
    +2013-03-01 09:11:58.70307 92.96 95.04000091552734
    +2013-03-01 09:11:58.70307 95.04 NULL
    +2013-03-01 09:11:58.703071 0.83 2.8600000143051147
    +2013-03-01 09:11:58.703071 1.99 6.294999837875366
    +2013-03-01 09:11:58.703071 3.73 9.739999771118164
    +2013-03-01 09:11:58.703071 8.86 10.96999979019165
    +2013-03-01 09:11:58.703071 10.62 12.074999809265137
    +2013-03-01 09:11:58.703071 11.32 13.764999866485596
    +2013-03-01 09:11:58.703071 12.83 14.829999923706055
    +2013-03-01 09:11:58.703071 14.7 16.269999980926514
    +2013-03-01 09:11:58.703071 14.96 18.34000015258789
    +2013-03-01 09:11:58.703071 17.58 20.05500030517578
    +2013-03-01 09:11:58.703071 19.1 23.980000495910645
    +2013-03-01 09:11:58.703071 21.01 27.09000015258789
    +2013-03-01 09:11:58.703071 26.95 28.149999618530273
    +2013-03-01 09:11:58.703071 27.23 29.389999389648438
    +2013-03-01 09:11:58.703071 29.07 30.774999618530273
    +2013-03-01 09:11:58.703071 29.71 31.890000343322754
    +2013-03-01 09:11:58.703071 31.84 33.63000011444092
    +2013-03-01 09:11:58.703071 31.94 36.31999969482422
    +2013-03-01 09:11:58.703071 35.32 37.90999984741211
    +2013-03-01 09:11:58.703071 37.32 40.290000915527344
    +2013-03-01 09:11:58.703071 38.5 43.19000053405762
    +2013-03-01 09:11:58.703071 42.08 44.47999954223633
    +2013-03-01 09:11:58.703071 44.3 45.75
    +2013-03-01 09:11:58.703071 44.66 47.864999771118164
    +2013-03-01 09:11:58.703071 46.84 49.26499938964844
    +2013-03-01 09:11:58.703071 48.89 49.959999084472656
    +2013-03-01 09:11:58.703071 49.64 51.18499946594238
    +2013-03-01 09:11:58.703071 50.28 52.67499923706055
    +2013-03-01 09:11:58.703071 52.09 53.67499923706055
    +2013-03-01 09:11:58.703071 53.26 55.27000045776367
    +2013-03-01 09:11:58.703071 54.09 56.60499954223633
    +2013-03-01 09:11:58.703071 56.45 59.084999084472656
    +2013-03-01 09:11:58.703071 56.76 61.64500045776367
    +2013-03-01 09:11:58.703071 61.41 62.454999923706055
    +2013-03-01 09:11:58.703071 61.88 63.790000915527344
    +2013-03-01 09:11:58.703071 63.03 66.58500289916992
    +2013-03-01 09:11:58.703071 64.55 72.375
    +2013-03-01 09:11:58.703071 68.62 77.59000015258789
    +2013-03-01 09:11:58.703071 76.13 79.7400016784668
    +2013-03-01 09:11:58.703071 79.05 80.92000198364258
    +2013-03-01 09:11:58.703071 80.43 82.13000106811523
    +2013-03-01 09:11:58.703071 81.41 83.41500091552734
    +2013-03-01 09:11:58.703071 82.85 84.09500122070312
    +2013-03-01 09:11:58.703071 83.98 84.88000106811523
    +2013-03-01 09:11:58.703071 84.21 86.7400016784668
    +2013-03-01 09:11:58.703071 85.55 88.43000030517578
    +2013-03-01 09:11:58.703071 87.93 91.5999984741211
    +2013-03-01 09:11:58.703071 88.93 96.8599967956543
    +2013-03-01 09:11:58.703071 94.27 99.44999694824219
    +2013-03-01 09:11:58.703071 99.45 NULL
    +2013-03-01 09:11:58.703072 0.36 0.635000005364418
    +2013-03-01 09:11:58.703072 0.48 1.030000001192093
    +2013-03-01 09:11:58.703072 0.79 2.875
    +2013-03-01 09:11:58.703072 1.27 6.740000009536743
    +2013-03-01 09:11:58.703072 4.48 16.135000228881836
    +2013-03-01 09:11:58.703072 9.0 24.199999809265137
    +2013-03-01 09:11:58.703072 23.27 25.234999656677246
    +2013-03-01 09:11:58.703072 25.13 25.625
    +2013-03-01 09:11:58.703072 25.34 27.460000038146973
    +2013-03-01 09:11:58.703072 25.91 29.739999771118164
    +2013-03-01 09:11:58.703072 29.01 34.21000003814697
    +2013-03-01 09:11:58.703072 30.47 38.625
    +2013-03-01 09:11:58.703072 37.95 42.60499954223633
    +2013-03-01 09:11:58.703072 39.3 49.17499923706055
    +2013-03-01 09:11:58.703072 45.91 53.26999855041504
    +2013-03-01 09:11:58.703072 52.44 55.39999961853027
    +2013-03-01 09:11:58.703072 54.1 57.73500061035156
    +2013-03-01 09:11:58.703072 56.7 60.43000030517578
    +2013-03-01 09:11:58.703072 58.77 65.14499855041504
    +2013-03-01 09:11:58.703072 62.09 69.93999862670898
    +2013-03-01 09:11:58.703072 68.2 75.56999969482422
    +2013-03-01 09:11:58.703072 71.68 79.73999786376953
    +2013-03-01 09:11:58.703072 79.46 80.65499877929688
    +2013-03-01 09:11:58.703072 80.02 84.13500213623047
    +PREHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +POSTHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@over10k
    +#### A masked pattern was here ####
    +2013-03-01 09:11:58.70307 3.17 7.03000020980835
    +2013-03-01 09:11:58.70307 10.89 9.533333460489908
    +2013-03-01 09:11:58.70307 14.54 10.84500002861023
    +2013-03-01 09:11:58.70307 14.78 12.24600009918213
    +2013-03-01 09:11:58.70307 17.85 13.640000184377035
    +2013-03-01 09:11:58.70307 20.61 15.790000234331403
    +2013-03-01 09:11:58.70307 28.69 17.46875011920929
    +2013-03-01 09:11:58.70307 29.22 18.99111122555203
    +2013-03-01 09:11:58.70307 31.17 20.926999950408934
    +2013-03-01 09:11:58.70307 38.35 22.534545464949176
    +2013-03-01 09:11:58.70307 38.61 23.94666663805644
    +2013-03-01 09:11:58.70307 39.48 25.22307696709266
    +2013-03-01 09:11:58.70307 40.54 26.39285707473755
    +2013-03-01 09:11:58.70307 41.6 27.70533339182536
    +2013-03-01 09:11:58.70307 46.08 29.371250092983246
    +2013-03-01 09:11:58.70307 54.36 30.992941183202408
    +2013-03-01 09:11:58.70307 56.94 32.8799999554952
    +2013-03-01 09:11:58.70307 64.96 35.01894714957789
    +2013-03-01 09:11:58.70307 73.52 37.196999883651735
    +2013-03-01 09:11:58.70307 78.58 39.30238101595924
    +2013-03-01 09:11:58.70307 81.41 41.36636365543712
    +2013-03-01 09:11:58.70307 84.71 43.369130466295324
    +2013-03-01 09:11:58.70307 87.43 45.368750055631004
    +2013-03-01 09:11:58.70307 91.36 47.272400016784665
    +2013-03-01 09:11:58.70307 92.96 49.10961543596708
    +2013-03-01 09:11:58.70307 95.04 49.10961543596708
    +2013-03-01 09:11:58.703071 0.83 1.4099999964237213
    +2013-03-01 09:11:58.703071 1.99 2.1833333373069763
    +2013-03-01 09:11:58.703071 3.73 3.8524999171495438
    +2013-03-01 09:11:58.703071 8.86 5.205999910831451
    +2013-03-01 09:11:58.703071 10.62 6.224999874830246
    +2013-03-01 09:11:58.703071 11.32 7.168571310383933
    +2013-03-01 09:11:58.703071 12.83 8.109999872744083
    +2013-03-01 09:11:58.703071 14.7 8.871111002233294
    +2013-03-01 09:11:58.703071 14.96 9.74199989438057
    +2013-03-01 09:11:58.703071 17.58 10.592727211388675
    +2013-03-01 09:11:58.703071 19.1 11.460833296179771
    +2013-03-01 09:11:58.703071 21.01 12.652307716699747
    +2013-03-01 09:11:58.703071 26.95 13.693571418523788
    +2013-03-01 09:11:58.703071 27.23 14.718666636943817
    +2013-03-01 09:11:58.703071 29.07 15.65562491491437
    +2013-03-01 09:11:58.703071 29.71 16.607646987718695
    +2013-03-01 09:11:58.703071 31.84 17.459444406959747
    +2013-03-01 09:11:58.703071 31.94 18.399473632636823
    +2013-03-01 09:11:58.703071 35.32 19.345499935746194
    +2013-03-01 09:11:58.703071 37.32 20.257618986424944
    +2013-03-01 09:11:58.703071 38.5 21.249545479362663
    +2013-03-01 09:11:58.703071 42.08 22.251739121001698
    +2013-03-01 09:11:58.703071 44.3 23.1854166512688
    +2013-03-01 09:11:58.703071 44.66 24.131599991321565
    +2013-03-01 09:11:58.703071 46.84 25.083846122026443
    +2013-03-01 09:11:58.703071 48.89 25.993333280086517
    +2013-03-01 09:11:58.703071 49.64 26.8607141907726
    +2013-03-01 09:11:58.703071 50.28 27.73068956876623
    +2013-03-01 09:11:58.703071 52.09 28.581666527191796
    +2013-03-01 09:11:58.703071 53.26 29.40451599897877
    +2013-03-01 09:11:58.703071 54.09 30.24968739785254
    +2013-03-01 09:11:58.703071 56.45 31.05303015311559
    +2013-03-01 09:11:58.703071 56.76 31.945882202947843
    +2013-03-01 09:11:58.703071 61.41 32.801142741952624
    +2013-03-01 09:11:58.703071 61.88 33.64083318743441
    +2013-03-01 09:11:58.703071 63.03 34.47621615674045
    +2013-03-01 09:11:58.703071 64.55 35.37473685647312
    +2013-03-01 09:11:58.703071 68.62 36.419743533317856
    +2013-03-01 09:11:58.703071 76.13 37.48550002127886
    +2013-03-01 09:11:58.703071 79.05 38.532926857471466
    +2013-03-01 09:11:58.703071 80.43 39.553809638534275
    +2013-03-01 09:11:58.703071 81.41 40.56069775098978
    +2013-03-01 09:11:58.703071 82.85 41.547500151124865
    +2013-03-01 09:11:58.703071 83.98 42.49555568297704
    +2013-03-01 09:11:58.703071 84.21 43.43152193012445
    +2013-03-01 09:11:58.703071 85.55 44.378298065763836
    +2013-03-01 09:11:58.703071 87.93 45.30645852908492
    +2013-03-01 09:11:58.703071 88.93 46.305714408962096
    +2013-03-01 09:11:58.703071 94.27 47.3686000597477
    +2013-03-01 09:11:58.703071 99.45 47.3686000597477
    +2013-03-01 09:11:58.703072 0.36 0.42000000178813934
    +2013-03-01 09:11:58.703072 0.48 0.5433333416779836
    +2013-03-01 09:11:58.703072 0.79 0.7250000014901161
    +2013-03-01 09:11:58.703072 1.27 1.47600000500679
    +2013-03-01 09:11:58.703072 4.48 2.730000004172325
    +2013-03-01 09:11:58.703072 9.0 5.664285783256803
    +2013-03-01 09:11:58.703072 23.27 8.097499955445528
    +2013-03-01 09:11:58.703072 25.13 10.013333310683569
    +2013-03-01 09:11:58.703072 25.34 11.602999964356423
    +2013-03-01 09:11:58.703072 25.91 13.185454533858733
    +2013-03-01 09:11:58.703072 29.01 14.62583326548338
    +2013-03-01 09:11:58.703072 30.47 16.419999996056923
    +2013-03-01 09:11:58.703072 37.95 18.05428565612861
    +2013-03-01 09:11:58.703072 39.3 19.911333268880846
    +2013-03-01 09:11:58.703072 45.91 21.944374853745103
    +2013-03-01 09:11:58.703072 52.44 23.835882125531924
    +2013-03-01 09:11:58.703072 54.1 25.661666494276787
    +2013-03-01 09:11:58.703072 56.7 27.404210387091887
    +2013-03-01 09:11:58.703072 58.77 29.138499875366687
    +2013-03-01 09:11:58.703072 62.09 30.998571164551237
    +2013-03-01 09:11:58.703072 68.2 32.84772703457963
    +2013-03-01 09:11:58.703072 71.68 34.87434755848802
    +2013-03-01 09:11:58.703072 79.46 36.75541627034545
    +2013-03-01 09:11:58.703072 80.02 38.53679965615272
      PREHOOK: query: -- count
      select ts, f, count(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
      PREHOOK: type: QUERY
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10728 : deprecate unix_timestamp(void) and make it deterministic (Sergey Shelukhin, reveiwed by Ashutosh Chauhan(


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24d3307b
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24d3307b
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24d3307b

    Branch: refs/heads/hbase-metastore
    Commit: 24d3307be79d35d3a34c49014dfdd597112f9106
    Parents: 7556361
    Author: Sergey Shelukhin <sershe@apache.org>
    Authored: Tue Jun 2 18:37:48 2015 -0700
    Committer: Sergey Shelukhin <sershe@apache.org>
    Committed: Tue Jun 2 18:37:48 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/session/SessionState.java | 4 +--
      .../udf/generic/GenericUDFToUnixTimeStamp.java | 6 +++-
      .../ql/udf/generic/GenericUDFUnixTimeStamp.java | 36 ++++++++++++++------
      .../queries/clientpositive/autogen_colalias.q | 4 +--
      .../queries/clientpositive/udf_unix_timestamp.q | 8 +++--
      .../clientpositive/autogen_colalias.q.out | 12 +++----
      ql/src/test/results/clientpositive/udf5.q.out | 30 +++++-----------
      .../clientpositive/udf_unix_timestamp.q.out | 32 +++++++++++++++--
      8 files changed, 85 insertions(+), 47 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    index 37b6d6f..7930298 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    @@ -903,12 +903,12 @@ public class SessionState {
            return ((ss != null) && (ss.out != null)) ? ss.out : System.out;
          }

    - public PrintStream getInfoStream() {
    + public static PrintStream getInfoStream() {
            SessionState ss = SessionState.get();
            return ((ss != null) && (ss.info != null)) ? ss.info : getErrStream();
          }

    - public PrintStream getErrStream() {
    + public static PrintStream getErrStream() {
            SessionState ss = SessionState.get();
            return ((ss != null) && (ss.err != null)) ? ss.err : System.err;
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
    index 65a2297..4ab5389 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
    @@ -152,10 +152,14 @@ public class GenericUDFToUnixTimeStamp extends GenericUDF {
            return retValue;
          }
          Timestamp timestamp = inputTimestampOI.getPrimitiveJavaObject(arguments[0].get());
    - retValue.set(timestamp.getTime() / 1000);
    + setValueFromTs(retValue, timestamp);
          return retValue;
        }

    + protected static void setValueFromTs(LongWritable value, Timestamp timestamp) {
    + value.set(timestamp.getTime() / 1000);
    + }
    +
        @Override
        public String getDisplayString(String[] children) {
          StringBuilder sb = new StringBuilder(32);

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
    index 0720c55..c1b2a01 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
    @@ -18,23 +18,43 @@

      package org.apache.hadoop.hive.ql.udf.generic;

    +import java.io.PrintStream;
    +
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.hive.ql.exec.Description;
      import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
      import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.ql.session.SessionState;
    +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
      import org.apache.hadoop.hive.ql.udf.UDFType;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
    +import org.apache.hadoop.io.LongWritable;
    +

    -@UDFType(deterministic = false)
    +@UDFType(deterministic = true)
      @Description(name = "unix_timestamp",
    - value = "_FUNC_([date[, pattern]]) - Returns the UNIX timestamp",
    - extended = "Converts the current or specified time to number of seconds "
    - + "since 1970-01-01.")
    + value = "_FUNC_(date[, pattern]) - Converts the time to a number",
    + extended = "Converts the specified time to number of seconds "
    + + "since 1970-01-01. The _FUNC_(void) overload is deprecated, use current_timestamp.")
      public class GenericUDFUnixTimeStamp extends GenericUDFToUnixTimeStamp {
    -
    + private static final Log LOG = LogFactory.getLog(GenericUDFUnixTimeStamp.class);
    + private LongWritable currentTimestamp; // retValue is transient so store this separately.
        @Override
        protected void initializeInput(ObjectInspector[] arguments) throws UDFArgumentException {
          if (arguments.length > 0) {
            super.initializeInput(arguments);
    + } else {
    + if (currentTimestamp == null) {
    + currentTimestamp = new LongWritable(0);
    + setValueFromTs(currentTimestamp, SessionState.get().getQueryCurrentTimestamp());
    + String msg = "unix_timestamp(void) is deprecated. Use current_timestamp instead.";
    + LOG.warn(msg);
    + PrintStream stream = LogHelper.getInfoStream();
    + if (stream != null) {
    + stream.println(msg);
    + }
    + }
          }
        }

    @@ -45,10 +65,6 @@ public class GenericUDFUnixTimeStamp extends GenericUDFToUnixTimeStamp {

        @Override
        public Object evaluate(DeferredObject[] arguments) throws HiveException {
    - if (arguments.length == 0) {
    - retValue.set(System.currentTimeMillis() / 1000);
    - return retValue;
    - }
    - return super.evaluate(arguments);
    + return (arguments.length == 0) ? currentTimestamp : super.evaluate(arguments);
        }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/queries/clientpositive/autogen_colalias.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/autogen_colalias.q b/ql/src/test/queries/clientpositive/autogen_colalias.q
    index e247a73..dac0bf7 100644
    --- a/ql/src/test/queries/clientpositive/autogen_colalias.q
    +++ b/ql/src/test/queries/clientpositive/autogen_colalias.q
    @@ -1,6 +1,6 @@
      CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax';

    -create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
    +create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
      1,
      0)
       from src group by src.key;
    @@ -12,7 +12,7 @@ describe dest_grouped_old2;
      set hive.autogen.columnalias.prefix.label=column_;
      set hive.autogen.columnalias.prefix.includefuncname=true;

    -create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
    +create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
       (src.key +5) % 2,
      0)
      from src group by src.key;

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/udf_unix_timestamp.q b/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
    index 1664329..bb598c2 100644
    --- a/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
    +++ b/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
    @@ -21,9 +21,13 @@ SELECT
        unix_timestamp('2009 Mar 20 11:30:01 am', 'yyyy MMM dd h:mm:ss a')
      FROM oneline;

    +create table foo as SELECT
    + 'deprecated' as a,
    + unix_timestamp() as b
    +FROM oneline;
    +drop table foo;
    +
      SELECT
        'random_string',
        unix_timestamp('random_string')
      FROM oneline;
    -
    -

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/results/clientpositive/autogen_colalias.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/autogen_colalias.q.out b/ql/src/test/results/clientpositive/autogen_colalias.q.out
    index 7257aaa..5569b46 100644
    --- a/ql/src/test/results/clientpositive/autogen_colalias.q.out
    +++ b/ql/src/test/results/clientpositive/autogen_colalias.q.out
    @@ -4,7 +4,7 @@ PREHOOK: Output: test_max
      POSTHOOK: query: CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
      POSTHOOK: type: CREATEFUNCTION
      POSTHOOK: Output: test_max
    -PREHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
    +PREHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
      1,
      0)
       from src group by src.key
    @@ -12,7 +12,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
      PREHOOK: Input: default@src
      PREHOOK: Output: database:default
      PREHOOK: Output: default@dest_grouped_old1
    -POSTHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
    +POSTHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
      1,
      0)
       from src group by src.key
    @@ -33,7 +33,7 @@ c3 int
      c4 bigint
      c5 double
      c6 bigint
    -c7 bigint
    +c7 timestamp
      c8 int
      c9 int
      PREHOOK: query: create table dest_grouped_old2 as select distinct src.key from src
    @@ -53,7 +53,7 @@ POSTHOOK: query: describe dest_grouped_old2
      POSTHOOK: type: DESCTABLE
      POSTHOOK: Input: default@dest_grouped_old2
      key string
    -PREHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
    +PREHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
       (src.key +5) % 2,
      0)
      from src group by src.key
    @@ -61,7 +61,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
      PREHOOK: Input: default@src
      PREHOOK: Output: database:default
      PREHOOK: Output: default@dest_grouped_new1
    -POSTHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
    +POSTHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
       (src.key +5) % 2,
      0)
      from src group by src.key
    @@ -82,7 +82,7 @@ test_max_length_src__3 int
      count_src_value_4 bigint
      sin_count_src_value_5 double
      count_sin_src_value_6 bigint
    -unix_timestamp_7 bigint
    +current_timestamp_7 timestamp
      sum_if_value_10_valu_8 int
      if_src_key_10_src_ke_9 double
      PREHOOK: query: create table dest_grouped_new2 as select distinct src.key from src

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/results/clientpositive/udf5.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/udf5.q.out b/ql/src/test/results/clientpositive/udf5.q.out
    index 26cf3f1..860ebcb 100644
    --- a/ql/src/test/results/clientpositive/udf5.q.out
    +++ b/ql/src/test/results/clientpositive/udf5.q.out
    @@ -54,33 +54,21 @@ POSTHOOK: query: EXPLAIN
      SELECT from_unixtime(unix_timestamp('2010-01-13 11:57:40', 'yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss'), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) from dest1
      POSTHOOK: type: QUERY
      STAGE DEPENDENCIES:
    - Stage-1 is a root stage
    - Stage-0 depends on stages: Stage-1
    + Stage-0 is a root stage

      STAGE PLANS:
    - Stage: Stage-1
    - Map Reduce
    - Map Operator Tree:
    - TableScan
    - alias: dest1
    - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE
    - Select Operator
    - expressions: from_unixtime(unix_timestamp('2010-01-13 11:57:40','yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss') (type: string), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) (type: string)
    - outputColumnNames: _col0, _col1
    - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
    - File Output Operator
    - compressed: false
    - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
    - table:
    - input format: org.apache.hadoop.mapred.TextInputFormat
    - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    -
        Stage: Stage-0
          Fetch Operator
            limit: -1
            Processor Tree:
    - ListSink
    + TableScan
    + alias: dest1
    + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE
    + Select Operator
    + expressions: '01/13/10 11:57:40' (type: string), '2010-01-13 11:57:40' (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
    + ListSink

      PREHOOK: query: SELECT from_unixtime(unix_timestamp('2010-01-13 11:57:40', 'yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss'), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) from dest1
      PREHOOK: type: QUERY

    http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out b/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
    index e4ffa1a..c64379d 100644
    --- a/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
    +++ b/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
    @@ -2,13 +2,13 @@ PREHOOK: query: DESCRIBE FUNCTION unix_timestamp
      PREHOOK: type: DESCFUNCTION
      POSTHOOK: query: DESCRIBE FUNCTION unix_timestamp
      POSTHOOK: type: DESCFUNCTION
    -unix_timestamp([date[, pattern]]) - Returns the UNIX timestamp
    +unix_timestamp(date[, pattern]) - Converts the time to a number
      PREHOOK: query: DESCRIBE FUNCTION EXTENDED unix_timestamp
      PREHOOK: type: DESCFUNCTION
      POSTHOOK: query: DESCRIBE FUNCTION EXTENDED unix_timestamp
      POSTHOOK: type: DESCFUNCTION
    -unix_timestamp([date[, pattern]]) - Returns the UNIX timestamp
    -Converts the current or specified time to number of seconds since 1970-01-01.
    +unix_timestamp(date[, pattern]) - Converts the time to a number
    +Converts the specified time to number of seconds since 1970-01-01. The unix_timestamp(void) overload is deprecated, use current_timestamp.
      PREHOOK: query: create table oneline(key int, value string)
      PREHOOK: type: CREATETABLE
      PREHOOK: Output: database:default
    @@ -70,6 +70,32 @@ POSTHOOK: type: QUERY
      POSTHOOK: Input: default@oneline
      #### A masked pattern was here ####
      2009 Mar 20 11:30:01 am 1237573801
    +unix_timestamp(void) is deprecated. Use current_timestamp instead.
    +unix_timestamp(void) is deprecated. Use current_timestamp instead.
    +PREHOOK: query: create table foo as SELECT
    + 'deprecated' as a,
    + unix_timestamp() as b
    +FROM oneline
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: default@oneline
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@foo
    +POSTHOOK: query: create table foo as SELECT
    + 'deprecated' as a,
    + unix_timestamp() as b
    +FROM oneline
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: default@oneline
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@foo
    +PREHOOK: query: drop table foo
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@foo
    +PREHOOK: Output: default@foo
    +POSTHOOK: query: drop table foo
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@foo
    +POSTHOOK: Output: default@foo
      PREHOOK: query: SELECT
        'random_string',
        unix_timestamp('random_string')
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10550: Dynamic RDD caching optimization for HoS [Spark Branch] (Chengxiang reviewed by Xuefu)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/441d4c08
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/441d4c08
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/441d4c08

    Branch: refs/heads/hbase-metastore
    Commit: 441d4c081b8bc4fa0b7d7f65ef34816d005d41b4
    Parents: 37cccbc
    Author: chengxiang <chengxiang@apache.com>
    Authored: Thu May 28 13:31:26 2015 +0800
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Jun 1 14:04:41 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/spark/CacheTran.java | 54 ++++++++++++++++++++
      .../hadoop/hive/ql/exec/spark/MapTran.java | 17 +++---
      .../hadoop/hive/ql/exec/spark/ReduceTran.java | 17 +++---
      .../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 ++
      .../hive/ql/exec/spark/SparkPlanGenerator.java | 25 +++++++--
      .../hive/ql/exec/spark/SparkUtilities.java | 36 +++++++++++++
      6 files changed, 134 insertions(+), 18 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
    new file mode 100644
    index 0000000..5ec27ec
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
    @@ -0,0 +1,54 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.hadoop.hive.ql.exec.spark;
    +
    +import org.apache.hadoop.io.WritableComparable;
    +import org.apache.spark.api.java.JavaPairRDD;
    +import org.apache.spark.storage.StorageLevel;
    +
    +public abstract class CacheTran<KI extends WritableComparable, VI, KO extends WritableComparable, VO>
    + implements SparkTran<KI, VI, KO, VO> {
    + // whether to cache current RDD.
    + private boolean caching = false;
    + private JavaPairRDD<KO, VO> cachedRDD;
    +
    + protected CacheTran(boolean cache) {
    + this.caching = cache;
    + }
    +
    + @Override
    + public JavaPairRDD<KO, VO> transform(
    + JavaPairRDD<KI, VI> input) {
    + if (caching) {
    + if (cachedRDD == null) {
    + cachedRDD = doTransform(input);
    + cachedRDD.persist(StorageLevel.MEMORY_AND_DISK());
    + }
    + return cachedRDD;
    + } else {
    + return doTransform(input);
    + }
    + }
    +
    + public Boolean isCacheEnable() {
    + return caching;
    + }
    +
    + protected abstract JavaPairRDD<KO, VO> doTransform(JavaPairRDD<KI, VI> input);
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    index 2170243..2a18991 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
    @@ -22,12 +22,20 @@ import org.apache.hadoop.hive.ql.io.HiveKey;
      import org.apache.hadoop.io.BytesWritable;
      import org.apache.spark.api.java.JavaPairRDD;

    -public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey, BytesWritable> {
    +public class MapTran extends CacheTran<BytesWritable, BytesWritable, HiveKey, BytesWritable> {
        private HiveMapFunction mapFunc;
        private String name = "MapTran";

    + public MapTran() {
    + this(false);
    + }
    +
    + public MapTran(boolean cache) {
    + super(cache);
    + }
    +
        @Override
    - public JavaPairRDD<HiveKey, BytesWritable> transform(
    + public JavaPairRDD<HiveKey, BytesWritable> doTransform(
            JavaPairRDD<BytesWritable, BytesWritable> input) {
          return input.mapPartitionsToPair(mapFunc);
        }
    @@ -42,11 +50,6 @@ public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey,
        }

        @Override
    - public Boolean isCacheEnable() {
    - return null;
    - }
    -
    - @Override
        public void setName(String name) {
          this.name = name;
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    index e60dfac..3d56876 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
    @@ -22,12 +22,20 @@ import org.apache.hadoop.hive.ql.io.HiveKey;
      import org.apache.hadoop.io.BytesWritable;
      import org.apache.spark.api.java.JavaPairRDD;

    -public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, HiveKey, BytesWritable> {
    +public class ReduceTran extends CacheTran<HiveKey, Iterable<BytesWritable>, HiveKey, BytesWritable> {
        private HiveReduceFunction reduceFunc;
        private String name = "Reduce";

    + public ReduceTran() {
    + this(false);
    + }
    +
    + public ReduceTran(boolean caching) {
    + super(caching);
    + }
    +
        @Override
    - public JavaPairRDD<HiveKey, BytesWritable> transform(
    + public JavaPairRDD<HiveKey, BytesWritable> doTransform(
            JavaPairRDD<HiveKey, Iterable<BytesWritable>> input) {
          return input.mapPartitionsToPair(reduceFunc);
        }
    @@ -42,11 +50,6 @@ public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, H
        }

        @Override
    - public Boolean isCacheEnable() {
    - return null;
    - }
    -
    - @Override
        public void setName(String name) {
          this.name = name;
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    index ee5c78a..762f734 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
    @@ -88,6 +88,9 @@ public class SparkPlan {
          }

          perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_BUILD_RDD_GRAPH);
    + if (LOG.isDebugEnabled()) {
    + LOG.info("print generated spark rdd graph:\n" + SparkUtilities.rddGraphToString(finalRDD));
    + }
          return finalRDD;
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
    index 3f240f5..3a1eff8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
    @@ -102,7 +102,7 @@ public class SparkPlanGenerator {
          try {
            for (BaseWork work : sparkWork.getAllWork()) {
              perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName());
    - SparkTran tran = generate(work);
    + SparkTran tran = generate(work, sparkWork);
              SparkTran parentTran = generateParentTran(sparkPlan, sparkWork, work);
              sparkPlan.addTran(tran);
              sparkPlan.connect(parentTran, tran);
    @@ -206,18 +206,19 @@ public class SparkPlanGenerator {
          return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache);
        }

    - private SparkTran generate(BaseWork work) throws Exception {
    + private SparkTran generate(BaseWork work, SparkWork sparkWork) throws Exception {
          initStatsPublisher(work);
          JobConf newJobConf = cloneJobConf(work);
          checkSpecs(work, newJobConf);
          byte[] confBytes = KryoSerializer.serializeJobConf(newJobConf);
    + boolean caching = isCachingWork(work, sparkWork);
          if (work instanceof MapWork) {
    - MapTran mapTran = new MapTran();
    + MapTran mapTran = new MapTran(caching);
            HiveMapFunction mapFunc = new HiveMapFunction(confBytes, sparkReporter);
            mapTran.setMapFunction(mapFunc);
            return mapTran;
          } else if (work instanceof ReduceWork) {
    - ReduceTran reduceTran = new ReduceTran();
    + ReduceTran reduceTran = new ReduceTran(caching);
            HiveReduceFunction reduceFunc = new HiveReduceFunction(confBytes, sparkReporter);
            reduceTran.setReduceFunction(reduceFunc);
            return reduceTran;
    @@ -227,6 +228,22 @@ public class SparkPlanGenerator {
          }
        }

    + private boolean isCachingWork(BaseWork work, SparkWork sparkWork) {
    + boolean caching = true;
    + List<BaseWork> children = sparkWork.getChildren(work);
    + if (children.size() < 2) {
    + caching = false;
    + } else {
    + // do not cache this if its child RDD is intend to be cached.
    + for (BaseWork child : children) {
    + if (cloneToWork.containsKey(child)) {
    + caching = false;
    + }
    + }
    + }
    + return caching;
    + }
    +
        private void checkSpecs(BaseWork work, JobConf jc) throws Exception {
          Set<Operator<?>> opList = work.getAllOperators();
          for (Operator<?> op : opList) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    index c012af8..ef612bd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
    @@ -21,6 +21,7 @@ import java.io.File;
      import java.io.IOException;
      import java.net.URI;
      import java.net.URISyntaxException;
    +import java.util.Collection;
      import java.util.UUID;

      import org.apache.commons.io.FilenameUtils;
    @@ -33,7 +34,12 @@ import org.apache.hadoop.hive.ql.io.HiveKey;
      import org.apache.hadoop.hive.ql.metadata.HiveException;
      import org.apache.hadoop.hive.ql.session.SessionState;
      import org.apache.hadoop.io.BytesWritable;
    +import org.apache.spark.Dependency;
      import org.apache.spark.SparkConf;
    +import org.apache.spark.api.java.JavaPairRDD;
    +import org.apache.spark.rdd.RDD;
    +import org.apache.spark.rdd.UnionRDD;
    +import scala.collection.JavaConversions;

      /**
       * Contains utilities methods used as part of Spark tasks.
    @@ -116,4 +122,34 @@ public class SparkUtilities {
          SessionState.get().setSparkSession(sparkSession);
          return sparkSession;
        }
    +
    +
    + public static String rddGraphToString(JavaPairRDD rdd) {
    + StringBuilder sb = new StringBuilder();
    + rddToString(rdd.rdd(), sb, "");
    + return sb.toString();
    + }
    +
    + private static void rddToString(RDD rdd, StringBuilder sb, String offset) {
    + sb.append(offset).append(rdd.getClass().getCanonicalName()).append("[").append(rdd.hashCode()).append("]");
    + if (rdd.getStorageLevel().useMemory()) {
    + sb.append("(cached)");
    + }
    + sb.append("\n");
    + Collection<Dependency> dependencies = JavaConversions.asJavaCollection(rdd.dependencies());
    + if (dependencies != null) {
    + offset += "\t";
    + for (Dependency dependency : dependencies) {
    + RDD parentRdd = dependency.rdd();
    + rddToString(parentRdd, sb, offset);
    + }
    + } else if (rdd instanceof UnionRDD) {
    + UnionRDD unionRDD = (UnionRDD) rdd;
    + offset += "\t";
    + Collection<RDD> parentRdds = JavaConversions.asJavaCollection(unionRDD.rdds());
    + for (RDD parentRdd : parentRdds) {
    + rddToString(parentRdd, sb, offset);
    + }
    + }
    + }
      }
  • Gates at Jun 5, 2015 at 11:16 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/itests/src/test/resources/testconfiguration.properties
    ----------------------------------------------------------------------
    diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
    index fe6ee17..47a1107 100644
    --- a/itests/src/test/resources/testconfiguration.properties
    +++ b/itests/src/test/resources/testconfiguration.properties
    @@ -226,6 +226,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
        vector_inner_join.q,\
        vector_interval_1.q,\
        vector_interval_2.q,\
    + vector_interval_mapjoin.q,\
        vector_join30.q,\
        vector_join_filters.q,\
        vector_join_nulls.q,\

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
    index 6b95360..6654166 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
    @@ -35,6 +35,16 @@ import org.apache.hadoop.io.Writable;
      public abstract class ColumnVector {

        /*
    + * The current kinds of column vectors.
    + */
    + public static enum Type {
    + LONG,
    + DOUBLE,
    + BYTES,
    + DECIMAL
    + }
    +
    + /*
         * If hasNulls is true, then this array contains true if the value
         * is null, otherwise false. The array is always allocated, so a batch can be re-used
         * later and nulls added.

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
    index 8c4b6ea..6673509 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
    @@ -20,7 +20,12 @@ package org.apache.hadoop.hive.ql.exec.vector;

      import java.util.Arrays;

    +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
      import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
    +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;

      /**
       * Class to keep information on a set of typed vector columns. Used by
    @@ -117,28 +122,41 @@ public class VectorColumnSetInfo {

        protected void addKey(String outputType) throws HiveException {
          indexLookup[addIndex] = new KeyLookupHelper();
    - if (VectorizationContext.isIntFamily(outputType) ||
    - VectorizationContext.isDatetimeFamily(outputType)) {
    +
    + String typeName = VectorizationContext.mapTypeNameSynonyms(outputType);
    +
    + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
    + Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
    +
    + switch (columnVectorType) {
    + case LONG:
            longIndices[longIndicesIndex] = addIndex;
            indexLookup[addIndex].setLong(longIndicesIndex);
            ++longIndicesIndex;
    - } else if (VectorizationContext.isFloatFamily(outputType)) {
    + break;
    +
    + case DOUBLE:
            doubleIndices[doubleIndicesIndex] = addIndex;
            indexLookup[addIndex].setDouble(doubleIndicesIndex);
            ++doubleIndicesIndex;
    - } else if (VectorizationContext.isStringFamily(outputType) ||
    - outputType.equalsIgnoreCase("binary")) {
    + break;
    +
    + case BYTES:
            stringIndices[stringIndicesIndex]= addIndex;
            indexLookup[addIndex].setString(stringIndicesIndex);
            ++stringIndicesIndex;
    - } else if (VectorizationContext.isDecimalFamily(outputType)) {
    - decimalIndices[decimalIndicesIndex]= addIndex;
    - indexLookup[addIndex].setDecimal(decimalIndicesIndex);
    - ++decimalIndicesIndex;
    - }
    - else {
    - throw new HiveException("Unsuported vector output type: " + outputType);
    + break;
    +
    + case DECIMAL:
    + decimalIndices[decimalIndicesIndex]= addIndex;
    + indexLookup[addIndex].setDecimal(decimalIndicesIndex);
    + ++decimalIndicesIndex;
    + break;
    +
    + default:
    + throw new HiveException("Unexpected column vector type " + columnVectorType);
          }
    +
          addIndex++;
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
    index 0058141..f12bfde 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
    @@ -20,6 +20,12 @@ package org.apache.hadoop.hive.ql.exec.vector;

      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
    +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;

      /**
       * This class copies specified columns of a row from one VectorizedRowBatch to another.
    @@ -186,7 +192,7 @@ public class VectorCopyRow {
        private CopyRow[] subRowToBatchCopiersByValue;
        private CopyRow[] subRowToBatchCopiersByReference;

    - public void init(VectorColumnMapping columnMapping) {
    + public void init(VectorColumnMapping columnMapping) throws HiveException {
          int count = columnMapping.getCount();
          subRowToBatchCopiersByValue = new CopyRow[count];
          subRowToBatchCopiersByReference = new CopyRow[count];
    @@ -194,24 +200,35 @@ public class VectorCopyRow {
          for (int i = 0; i < count; i++) {
            int inputColumn = columnMapping.getInputColumns()[i];
            int outputColumn = columnMapping.getOutputColumns()[i];
    - String typeName = columnMapping.getTypeNames()[i];
    + String typeName = columnMapping.getTypeNames()[i].toLowerCase();
    + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
    + Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);

            CopyRow copyRowByValue = null;
            CopyRow copyRowByReference = null;

    - if (VectorizationContext.isIntFamily(typeName) ||
    - VectorizationContext.isDatetimeFamily(typeName)) {
    + switch (columnVectorType) {
    + case LONG:
              copyRowByValue = new LongCopyRow(inputColumn, outputColumn);
    - } else if (VectorizationContext.isFloatFamily(typeName)) {
    + break;
    +
    + case DOUBLE:
              copyRowByValue = new DoubleCopyRow(inputColumn, outputColumn);
    - } else if (VectorizationContext.isStringFamily(typeName)) {
    + break;
    +
    + case BYTES:
              copyRowByValue = new BytesCopyRowByValue(inputColumn, outputColumn);
              copyRowByReference = new BytesCopyRowByReference(inputColumn, outputColumn);
    - } else if (VectorizationContext.decimalTypePattern.matcher(typeName).matches()){
    + break;
    +
    + case DECIMAL:
              copyRowByValue = new DecimalCopyRow(inputColumn, outputColumn);
    - } else {
    - throw new RuntimeException("Cannot allocate vector copy row for " + typeName);
    + break;
    +
    + default:
    + throw new HiveException("Unexpected column vector type " + columnVectorType);
            }
    +
            subRowToBatchCopiersByValue[i] = copyRowByValue;
            if (copyRowByReference == null) {
              subRowToBatchCopiersByReference[i] = copyRowByValue;

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    index 61d2972..392e56d 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    @@ -107,6 +107,7 @@ import org.apache.hadoop.hive.ql.udf.generic.*;
      import org.apache.hadoop.hive.serde2.io.DateWritable;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
    +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
      import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
    @@ -2044,6 +2045,51 @@ public class VectorizationContext {
          }
        }

    + public static String mapTypeNameSynonyms(String typeName) {
    + typeName = typeName.toLowerCase();
    + if (typeName.equals("long")) {
    + return "bigint";
    + } else if (typeName.equals("string_family")) {
    + return "string";
    + } else {
    + return typeName;
    + }
    + }
    +
    + public static ColumnVector.Type getColumnVectorTypeFromTypeInfo(TypeInfo typeInfo) throws HiveException {
    + PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
    + PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
    +
    + switch (primitiveCategory) {
    + case BOOLEAN:
    + case BYTE:
    + case SHORT:
    + case INT:
    + case LONG:
    + case DATE:
    + case TIMESTAMP:
    + case INTERVAL_YEAR_MONTH:
    + case INTERVAL_DAY_TIME:
    + return ColumnVector.Type.LONG;
    +
    + case FLOAT:
    + case DOUBLE:
    + return ColumnVector.Type.DOUBLE;
    +
    + case STRING:
    + case CHAR:
    + case VARCHAR:
    + case BINARY:
    + return ColumnVector.Type.BYTES;
    +
    + case DECIMAL:
    + return ColumnVector.Type.DECIMAL;
    +
    + default:
    + throw new HiveException("Unexpected primitive type category " + primitiveCategory);
    + }
    + }
    +
        // TODO: When we support vectorized STRUCTs and can handle more in the reduce-side (MERGEPARTIAL):
        // TODO: Write reduce-side versions of AVG. Currently, only map-side (HASH) versions are in table.
        // TODO: And, investigate if different reduce-side versions are needed for var* and std*, or if map-side aggregate can be used.. Right now they are conservatively

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
    index af78776..4c8c4b1 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
    @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion;
      import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
      import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
      import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
    +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
      import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
      import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
      import org.apache.hadoop.hive.ql.exec.vector.mapjoin.optimized.VectorMapJoinOptimizedCreateHashTable;
    @@ -68,8 +69,12 @@ import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc;
      import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableImplementationType;
      import org.apache.hadoop.hive.ql.plan.api.OperatorType;
      import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
    +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
      import org.apache.hadoop.hive.serde2.objectinspector.StructField;
      import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
    +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;

      /**
       * This class is common operator class for native vectorized map join.
    @@ -658,7 +663,7 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
         * Setup our 2nd batch with the same "column schema" as the big table batch that can be used to
         * build join output results in.
         */
    - protected VectorizedRowBatch setupOverflowBatch() {
    + protected VectorizedRowBatch setupOverflowBatch() throws HiveException {
          VectorizedRowBatch overflowBatch;

          Map<Integer, String> scratchColumnTypeMap = vOutContext.getScratchColumnTypeMap();
    @@ -701,23 +706,38 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
         * Allocate overflow batch columns by hand.
         */
        private void allocateOverflowBatchColumnVector(VectorizedRowBatch overflowBatch, int outputColumn,
    - String typeName) {
    + String typeName) throws HiveException {

          if (overflowBatch.cols[outputColumn] == null) {
    - String vectorTypeName;
    - if (VectorizationContext.isIntFamily(typeName) ||
    - VectorizationContext.isDatetimeFamily(typeName)) {
    - vectorTypeName = "long";
    - } else if (VectorizationContext.isFloatFamily(typeName)) {
    - vectorTypeName = "double";
    - } else if (VectorizationContext.isStringFamily(typeName)) {
    - vectorTypeName = "string";
    - } else if (VectorizationContext.decimalTypePattern.matcher(typeName).matches()){
    - vectorTypeName = typeName; // Keep precision and scale.
    - } else {
    - throw new RuntimeException("Cannot determine vector type for " + typeName);
    + typeName = VectorizationContext.mapTypeNameSynonyms(typeName);
    +
    + String columnVectorTypeName;
    +
    + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
    + Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
    +
    + switch (columnVectorType) {
    + case LONG:
    + columnVectorTypeName = "long";
    + break;
    +
    + case DOUBLE:
    + columnVectorTypeName = "double";
    + break;
    +
    + case BYTES:
    + columnVectorTypeName = "string";
    + break;
    +
    + case DECIMAL:
    + columnVectorTypeName = typeName; // Keep precision and scale.
    + break;
    +
    + default:
    + throw new HiveException("Unexpected column vector type " + columnVectorType);
            }
    - overflowBatch.cols[outputColumn] = VectorizedRowBatchCtx.allocateColumnVector(vectorTypeName, VectorizedRowBatch.DEFAULT_SIZE);
    +
    + overflowBatch.cols[outputColumn] = VectorizedRowBatchCtx.allocateColumnVector(columnVectorTypeName, VectorizedRowBatch.DEFAULT_SIZE);

            if (LOG.isDebugEnabled()) {
              LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator initializeOp overflowBatch outputColumn " + outputColumn + " class " + overflowBatch.cols[outputColumn].getClass().getSimpleName());

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q b/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q
    new file mode 100644
    index 0000000..9a58658
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q
    @@ -0,0 +1,87 @@
    +SET hive.vectorized.execution.enabled=true;
    +SET hive.auto.convert.join=true;
    +
    +create table vectortab_a_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE;
    +
    +LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k;
    +
    +CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k;
    +
    +create table vectortab_b_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE;
    +
    +LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k;
    +
    +CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k;
    +
    +explain
    +select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s;
    +
    +select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s;
    \ No newline at end of file

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out
    new file mode 100644
    index 0000000..b4d3477
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out
    @@ -0,0 +1,281 @@
    +PREHOOK: query: create table vectortab_a_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_a_1k
    +POSTHOOK: query: create table vectortab_a_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_a_1k
    +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@vectortab_a_1k
    +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@vectortab_a_1k
    +PREHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: default@vectortab_a_1k
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_a_1korc
    +POSTHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: default@vectortab_a_1k
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_a_1korc
    +PREHOOK: query: create table vectortab_b_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_b_1k
    +POSTHOOK: query: create table vectortab_b_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_b_1k
    +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@vectortab_b_1k
    +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@vectortab_b_1k
    +PREHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: default@vectortab_b_1k
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_b_1korc
    +POSTHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: default@vectortab_b_1k
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_b_1korc
    +PREHOOK: query: explain
    +select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Tez
    + Edges:
    + Map 1 <- Map 2 (BROADCAST_EDGE)
    +#### A masked pattern was here ####
    + Vertices:
    + Map 1
    + Map Operator Tree:
    + TableScan
    + alias: vectortab_a_1korc
    + Statistics: Num rows: 1000 Data size: 460264 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: s is not null (type: boolean)
    + Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: _col1 is not null (type: boolean)
    + Statistics: Num rows: 250 Data size: 115066 Basic stats: COMPLETE Column stats: NONE
    + Map Join Operator
    + condition map:
    + Inner Join 0 to 1
    + keys:
    + 0 _col1 (type: interval_day_time), _col0 (type: string)
    + 1 _col1 (type: interval_day_time), _col0 (type: string)
    + outputColumnNames: _col0, _col1, _col2
    + input vertices:
    + 1 Map 2
    + Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
    + HybridGraceHashJoin: true
    + Select Operator
    + expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + Execution mode: vectorized
    + Map 2
    + Map Operator Tree:
    + TableScan
    + alias: vectortab_b_1korc
    + Statistics: Num rows: 1000 Data size: 458448 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: s is not null (type: boolean)
    + Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: _col1 is not null (type: boolean)
    + Statistics: Num rows: 250 Data size: 114612 Basic stats: COMPLETE Column stats: NONE
    + Reduce Output Operator
    + key expressions: _col1 (type: interval_day_time), _col0 (type: string)
    + sort order: ++
    + Map-reduce partition columns: _col1 (type: interval_day_time), _col0 (type: string)
    + Statistics: Num rows: 250 Data size: 114612 Basic stats: COMPLETE Column stats: NONE
    + Execution mode: vectorized
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@vectortab_a_1korc
    +PREHOOK: Input: default@vectortab_b_1korc
    +#### A masked pattern was here ####
    +POSTHOOK: query: select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@vectortab_a_1korc
    +POSTHOOK: Input: default@vectortab_b_1korc
    +#### A masked pattern was here ####

    http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out b/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
    new file mode 100644
    index 0000000..976091b
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
    @@ -0,0 +1,281 @@
    +PREHOOK: query: create table vectortab_a_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_a_1k
    +POSTHOOK: query: create table vectortab_a_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_a_1k
    +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@vectortab_a_1k
    +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@vectortab_a_1k
    +PREHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: default@vectortab_a_1k
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_a_1korc
    +POSTHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: default@vectortab_a_1k
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_a_1korc
    +PREHOOK: query: create table vectortab_b_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_b_1k
    +POSTHOOK: query: create table vectortab_b_1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + dc decimal(38,18),
    + bo boolean,
    + s string,
    + s2 string,
    + ts timestamp,
    + ts2 timestamp,
    + dt date)
    +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
    +STORED AS TEXTFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_b_1k
    +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@vectortab_b_1k
    +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@vectortab_b_1k
    +PREHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: default@vectortab_b_1k
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@vectortab_b_1korc
    +POSTHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: default@vectortab_b_1k
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@vectortab_b_1korc
    +PREHOOK: query: explain
    +select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-4 is a root stage
    + Stage-3 depends on stages: Stage-4
    + Stage-0 depends on stages: Stage-3
    +
    +STAGE PLANS:
    + Stage: Stage-4
    + Map Reduce Local Work
    + Alias -> Map Local Tables:
    + v2:vectortab_b_1korc
    + Fetch Operator
    + limit: -1
    + Alias -> Map Local Operator Tree:
    + v2:vectortab_b_1korc
    + TableScan
    + alias: vectortab_b_1korc
    + Statistics: Num rows: 1000 Data size: 458448 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: s is not null (type: boolean)
    + Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: _col1 is not null (type: boolean)
    + Statistics: Num rows: 250 Data size: 114612 Basic stats: COMPLETE Column stats: NONE
    + HashTable Sink Operator
    + keys:
    + 0 _col1 (type: interval_day_time), _col0 (type: string)
    + 1 _col1 (type: interval_day_time), _col0 (type: string)
    +
    + Stage: Stage-3
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: vectortab_a_1korc
    + Statistics: Num rows: 1000 Data size: 460264 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: s is not null (type: boolean)
    + Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: _col1 is not null (type: boolean)
    + Statistics: Num rows: 250 Data size: 115066 Basic stats: COMPLETE Column stats: NONE
    + Map Join Operator
    + condition map:
    + Inner Join 0 to 1
    + keys:
    + 0 _col1 (type: interval_day_time), _col0 (type: string)
    + 1 _col1 (type: interval_day_time), _col0 (type: string)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + Local Work:
    + Map Reduce Local Work
    + Execution mode: vectorized
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@vectortab_a_1korc
    +PREHOOK: Input: default@vectortab_b_1korc
    +#### A masked pattern was here ####
    +POSTHOOK: query: select
    + v1.s,
    + v2.s,
    + v1.intrvl1
    +from
    + ( select
    + s,
    + (cast(dt as date) - cast(ts as date)) as intrvl1
    + from
    + vectortab_a_1korc ) v1
    +join
    + (
    + select
    + s ,
    + (cast(dt as date) - cast(ts as date)) as intrvl2
    + from
    + vectortab_b_1korc
    + ) v2
    + on v1.intrvl1 = v2.intrvl2
    + and v1.s = v2.s
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@vectortab_a_1korc
    +POSTHOOK: Input: default@vectortab_b_1korc
    +#### A masked pattern was here ####
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10302 Load small tables (for map join) in executor memory only once [Spark Branch] (added new file missed in rebasing)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9bc8ea2a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9bc8ea2a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9bc8ea2a

    Branch: refs/heads/hbase-metastore
    Commit: 9bc8ea2ae25e2ef766604068ac143eeeaa376929
    Parents: 3817bb5
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Mon Jun 1 15:59:32 2015 -0700
    Committer: Jimmy Xiang <jxiang@cloudera.com>
    Committed: Mon Jun 1 15:59:32 2015 -0700

    ----------------------------------------------------------------------
      .../hive/ql/exec/spark/SmallTableCache.java | 73 ++++++++++++++++++++
      1 file changed, 73 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/9bc8ea2a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java
    new file mode 100644
    index 0000000..1992d16
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java
    @@ -0,0 +1,73 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.exec.spark;
    +
    +import java.util.concurrent.ConcurrentHashMap;
    +
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.conf.Configuration;
    +import org.apache.hadoop.fs.Path;
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
    +
    +public class SmallTableCache {
    + private static final Log LOG = LogFactory.getLog(SmallTableCache.class.getName());
    +
    + private static final ConcurrentHashMap<Path, MapJoinTableContainer>
    + tableContainerMap = new ConcurrentHashMap<Path, MapJoinTableContainer>();
    + private static volatile String queryId;
    +
    + /**
    + * Check if this is a new query. If so, clean up the cache
    + * that is for the previous query, and reset the current query id.
    + */
    + public static void initialize(Configuration conf) {
    + String currentQueryId = conf.get(HiveConf.ConfVars.HIVEQUERYID.varname);
    + if (!currentQueryId.equals(queryId)) {
    + if (!tableContainerMap.isEmpty()) {
    + synchronized (tableContainerMap) {
    + if (!currentQueryId.equals(queryId) && !tableContainerMap.isEmpty()) {
    + for (MapJoinTableContainer tableContainer: tableContainerMap.values()) {
    + tableContainer.clear();
    + }
    + tableContainerMap.clear();
    + if (LOG.isDebugEnabled()) {
    + LOG.debug("Cleaned up small table cache for query " + queryId);
    + }
    + }
    + }
    + }
    + queryId = currentQueryId;
    + }
    + }
    +
    + public static void cache(Path path, MapJoinTableContainer tableContainer) {
    + if (tableContainerMap.putIfAbsent(path, tableContainer) == null && LOG.isDebugEnabled()) {
    + LOG.debug("Cached small table file " + path + " for query " + queryId);
    + }
    + }
    +
    + public static MapJoinTableContainer get(Path path) {
    + MapJoinTableContainer tableContainer = tableContainerMap.get(path);
    + if (tableContainer != null && LOG.isDebugEnabled()) {
    + LOG.debug("Loaded small table file " + path + " from cache for query " + queryId);
    + }
    + return tableContainer;
    + }
    +}
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10887 : TestCliDriver tests ordering issues with Mac and CentOS (Hari Subramaniyan, reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4f57d88
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4f57d88
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4f57d88

    Branch: refs/heads/hbase-metastore
    Commit: b4f57d8885a85a8d977fa48fa756c5c2b1cd2be7
    Parents: 0910083
    Author: Hari Subramaniyan <harisankar@apache.org>
    Authored: Wed Jun 3 13:30:50 2015 -0700
    Committer: Hari Subramaniyan <harisankar@apache.org>
    Committed: Wed Jun 3 13:30:50 2015 -0700

    ----------------------------------------------------------------------
      .../queries/clientpositive/unionDistinct_2.q | 2 +
      .../update_after_multiple_inserts.q | 2 +
      .../clientpositive/tez/unionDistinct_2.q.out | 38 ++++++++++--------
      .../tez/update_after_multiple_inserts.q.out | 12 ++++--
      .../clientpositive/unionDistinct_2.q.out | 42 +++++++++++---------
      .../update_after_multiple_inserts.q.out | 12 ++++--
      6 files changed, 64 insertions(+), 44 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/queries/clientpositive/unionDistinct_2.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/unionDistinct_2.q b/ql/src/test/queries/clientpositive/unionDistinct_2.q
    index ab78128..324bf68 100644
    --- a/ql/src/test/queries/clientpositive/unionDistinct_2.q
    +++ b/ql/src/test/queries/clientpositive/unionDistinct_2.q
    @@ -1,3 +1,5 @@
    +-- SORT_QUERY_RESULTS
    +
      CREATE TABLE u1 as select key, value from src order by key limit 5;

      CREATE TABLE u2 as select key, value from src order by key limit 3;

    http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q b/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
    index 6ef209f..eb8e5a1 100644
    --- a/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
    +++ b/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
    @@ -3,6 +3,8 @@ set hive.support.concurrency=true;
      set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
      set hive.enforce.bucketing=true;

    +-- SORT_QUERY_RESULTS
    +
      create table acid_uami(i int,
                       de decimal(5,2),
                       vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');

    http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
    index 1ba78b7..6d59369 100644
    --- a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
    +++ b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
    @@ -1,9 +1,13 @@
    -PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
    +PREHOOK: query: -- SORT_QUERY_RESULTS
    +
    +CREATE TABLE u1 as select key, value from src order by key limit 5
      PREHOOK: type: CREATETABLE_AS_SELECT
      PREHOOK: Input: default@src
      PREHOOK: Output: database:default
      PREHOOK: Output: default@u1
    -POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
    +POSTHOOK: query: -- SORT_QUERY_RESULTS
    +
    +CREATE TABLE u1 as select key, value from src order by key limit 5
      POSTHOOK: type: CREATETABLE_AS_SELECT
      POSTHOOK: Input: default@src
      POSTHOOK: Output: database:default
    @@ -60,11 +64,11 @@ POSTHOOK: query: select * from u3
      POSTHOOK: type: QUERY
      POSTHOOK: Input: default@u3
      #### A masked pattern was here ####
    -98 val_98
    -98 val_98
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -94,16 +98,16 @@ POSTHOOK: Input: default@u3
      0 val_0
      0 val_0
      0 val_0
    -10 val_10
    -100 val_100
      0 val_0
      0 val_0
      0 val_0
    -98 val_98
    -98 val_98
    +10 val_10
    +100 val_100
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -133,11 +137,11 @@ POSTHOOK: Input: default@u3
      0 val_0
      10 val_10
      100 val_100
    -98 val_98
    -98 val_98
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -167,11 +171,11 @@ POSTHOOK: Input: default@u3
      0 val_0
      10 val_10
      100 val_100
    -98 val_98
    -98 val_98
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -504,9 +508,9 @@ POSTHOOK: Input: default@u2
      POSTHOOK: Input: default@v
      #### A masked pattern was here ####
      0 val_0
    +0 val_0
      10 val_10
      100 val_100
    -0 val_0
      PREHOOK: query: select distinct * from
      (
      select key, value from u1

    http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out b/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
    index a2ad3af..d536fe9 100644
    --- a/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
    +++ b/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
    @@ -1,10 +1,14 @@
    -PREHOOK: query: create table acid_uami(i int,
    +PREHOOK: query: -- SORT_QUERY_RESULTS
    +
    +create table acid_uami(i int,
                       de decimal(5,2),
                       vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
      PREHOOK: type: CREATETABLE
      PREHOOK: Output: database:default
      PREHOOK: Output: default@acid_uami
    -POSTHOOK: query: create table acid_uami(i int,
    +POSTHOOK: query: -- SORT_QUERY_RESULTS
    +
    +create table acid_uami(i int,
                       de decimal(5,2),
                       vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
      POSTHOOK: type: CREATETABLE
    @@ -50,8 +54,8 @@ POSTHOOK: Input: default@acid_uami
      #### A masked pattern was here ####
      1 109.23 mary had a little lamb
      10 119.23 and everywhere that mary went
    -65530 823.19 the lamb was sure to go
      6553 923.19 its fleece was white as snow
    +65530 823.19 the lamb was sure to go
      PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
      PREHOOK: type: QUERY
      PREHOOK: Input: default@acid_uami
    @@ -70,5 +74,5 @@ POSTHOOK: Input: default@acid_uami
      #### A masked pattern was here ####
      1 3.14 mary had a little lamb
      10 3.14 and everywhere that mary went
    -65530 823.19 the lamb was sure to go
      6553 923.19 its fleece was white as snow
    +65530 823.19 the lamb was sure to go

    http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/unionDistinct_2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/unionDistinct_2.q.out
    index 66df734..6d59369 100644
    --- a/ql/src/test/results/clientpositive/unionDistinct_2.q.out
    +++ b/ql/src/test/results/clientpositive/unionDistinct_2.q.out
    @@ -1,9 +1,13 @@
    -PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
    +PREHOOK: query: -- SORT_QUERY_RESULTS
    +
    +CREATE TABLE u1 as select key, value from src order by key limit 5
      PREHOOK: type: CREATETABLE_AS_SELECT
      PREHOOK: Input: default@src
      PREHOOK: Output: database:default
      PREHOOK: Output: default@u1
    -POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
    +POSTHOOK: query: -- SORT_QUERY_RESULTS
    +
    +CREATE TABLE u1 as select key, value from src order by key limit 5
      POSTHOOK: type: CREATETABLE_AS_SELECT
      POSTHOOK: Input: default@src
      POSTHOOK: Output: database:default
    @@ -60,11 +64,11 @@ POSTHOOK: query: select * from u3
      POSTHOOK: type: QUERY
      POSTHOOK: Input: default@u3
      #### A masked pattern was here ####
    -98 val_98
    -98 val_98
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -91,19 +95,19 @@ POSTHOOK: Input: default@u1
      POSTHOOK: Input: default@u2
      POSTHOOK: Input: default@u3
      #### A masked pattern was here ####
    -98 val_98
    -98 val_98
    -97 val_97
    -97 val_97
    -96 val_96
      0 val_0
      0 val_0
      0 val_0
    -10 val_10
    -100 val_100
      0 val_0
      0 val_0
      0 val_0
    +10 val_10
    +100 val_100
    +96 val_96
    +97 val_97
    +97 val_97
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -133,11 +137,11 @@ POSTHOOK: Input: default@u3
      0 val_0
      10 val_10
      100 val_100
    -98 val_98
    -98 val_98
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -167,11 +171,11 @@ POSTHOOK: Input: default@u3
      0 val_0
      10 val_10
      100 val_100
    -98 val_98
    -98 val_98
    +96 val_96
      97 val_97
      97 val_97
    -96 val_96
    +98 val_98
    +98 val_98
      PREHOOK: query: select key, value from
      (
      select key, value from u1
    @@ -504,9 +508,9 @@ POSTHOOK: Input: default@u2
      POSTHOOK: Input: default@v
      #### A masked pattern was here ####
      0 val_0
    +0 val_0
      10 val_10
      100 val_100
    -0 val_0
      PREHOOK: query: select distinct * from
      (
      select key, value from u1

    http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out b/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
    index a2ad3af..d536fe9 100644
    --- a/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
    +++ b/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
    @@ -1,10 +1,14 @@
    -PREHOOK: query: create table acid_uami(i int,
    +PREHOOK: query: -- SORT_QUERY_RESULTS
    +
    +create table acid_uami(i int,
                       de decimal(5,2),
                       vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
      PREHOOK: type: CREATETABLE
      PREHOOK: Output: database:default
      PREHOOK: Output: default@acid_uami
    -POSTHOOK: query: create table acid_uami(i int,
    +POSTHOOK: query: -- SORT_QUERY_RESULTS
    +
    +create table acid_uami(i int,
                       de decimal(5,2),
                       vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
      POSTHOOK: type: CREATETABLE
    @@ -50,8 +54,8 @@ POSTHOOK: Input: default@acid_uami
      #### A masked pattern was here ####
      1 109.23 mary had a little lamb
      10 119.23 and everywhere that mary went
    -65530 823.19 the lamb was sure to go
      6553 923.19 its fleece was white as snow
    +65530 823.19 the lamb was sure to go
      PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
      PREHOOK: type: QUERY
      PREHOOK: Input: default@acid_uami
    @@ -70,5 +74,5 @@ POSTHOOK: Input: default@acid_uami
      #### A masked pattern was here ####
      1 3.14 mary had a little lamb
      10 3.14 and everywhere that mary went
    -65530 823.19 the lamb was sure to go
      6553 923.19 its fleece was white as snow
    +65530 823.19 the lamb was sure to go
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10845: TezJobMonitor uses killedTaskCount instead of killedTaskAttemptCount (Sid Seth via Gunther Hagleitner)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e22aaf63
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e22aaf63
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e22aaf63

    Branch: refs/heads/hbase-metastore
    Commit: e22aaf6381a607a39bb238e5376ca3928790fc08
    Parents: b6b638a
    Author: Gunther Hagleitner <gunther@apache.org>
    Authored: Mon Jun 1 17:05:27 2015 -0700
    Committer: Gunther Hagleitner <gunther@apache.org>
    Committed: Mon Jun 1 17:05:27 2015 -0700

    ----------------------------------------------------------------------
      .../java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java | 6 +++---
      1 file changed, 3 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/e22aaf63/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    index 4423cd1..78caba8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    @@ -496,7 +496,7 @@ public class TezJobMonitor {
            if (progress != null) {
              final int totalTasks = progress.getTotalTaskCount();
              final int failedTaskAttempts = progress.getFailedTaskAttemptCount();
    - final int killedTasks = progress.getKilledTaskCount();
    + final int killedTaskAttempts = progress.getKilledTaskAttemptCount();
              final double duration =
                  perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName) / 1000.0;
              VertexStatus vertexStatus = null;
    @@ -583,7 +583,7 @@ public class TezJobMonitor {
                  vertexName,
                  totalTasks,
                  failedTaskAttempts,
    - killedTasks,
    + killedTaskAttempts,
                  secondsFormat.format((duration)),
                  commaFormat.format(cpuTimeMillis),
                  commaFormat.format(gcTimeMillis),
    @@ -623,7 +623,7 @@ public class TezJobMonitor {
            final int failed = progress.getFailedTaskAttemptCount();
            final int pending = progress.getTotalTaskCount() - progress.getSucceededTaskCount() -
                progress.getRunningTaskCount();
    - final int killed = progress.getKilledTaskCount();
    + final int killed = progress.getKilledTaskAttemptCount();

            // To get vertex status we can use DAGClient.getVertexStatus(), but it will be expensive to
            // get status from AM for every refresh of the UI. Lets infer the state from task counts.
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10705 Update tests for HIVE-9302 after removing binaries(Ferdinand Xu, reviewed by Hari Sankar Sivarama Subramaniyan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aec98e9f
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aec98e9f
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aec98e9f

    Branch: refs/heads/hbase-metastore
    Commit: aec98e9f1b9d3bd0926b0b84c6ed49bbe494667a
    Parents: 4140da5
    Author: Ferdinand Xu <cheng.a.xu@intel.com>
    Authored: Wed Jun 3 14:02:13 2015 -0400
    Committer: Ferdinand Xu <cheng.a.xu@intel.com>
    Committed: Wed Jun 3 14:02:20 2015 -0400

    ----------------------------------------------------------------------
      beeline/pom.xml | 6 ++
      .../hive/beeline/TestBeelineArgParsing.java | 32 +++++-----
      .../test/resources/DummyDriver-1.0-SNAPSHOT.jar | Bin 2812 -> 0 bytes
      beeline/src/test/resources/DummyDriver.txt | 59 +++++++++++++++++++
      .../src/test/resources/postgresql-9.3.jdbc3.jar | Bin 539003 -> 0 bytes
      .../apache/hive/common/util/HiveTestUtils.java | 46 +++++++++++++++
      .../hive/ql/session/TestSessionState.java | 40 ++-----------
      7 files changed, 133 insertions(+), 50 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/pom.xml
    ----------------------------------------------------------------------
    diff --git a/beeline/pom.xml b/beeline/pom.xml
    index 352f561..4ead3d3 100644
    --- a/beeline/pom.xml
    +++ b/beeline/pom.xml
    @@ -111,6 +111,12 @@
            <version>${junit.version}</version>
            <scope>test</scope>
          </dependency>
    + <dependency>
    + <groupId>postgresql</groupId>
    + <artifactId>postgresql</artifactId>
    + <version>9.1-901.jdbc4</version>
    + <scope>test</scope>
    + </dependency>
        </dependencies>

        <profiles>

    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
    ----------------------------------------------------------------------
    diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
    index 7a354f3..702805f 100644
    --- a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
    +++ b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
    @@ -18,6 +18,7 @@

      package org.apache.hive.beeline;

    +import java.io.IOException;
      import java.util.ArrayList;
      import java.util.Arrays;
      import java.util.Collection;
    @@ -30,6 +31,7 @@ import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
      import org.apache.hive.common.util.HiveTestUtils;
      import org.junit.Assert;
    +import org.junit.Before;
      import org.junit.Test;
      import org.junit.runner.RunWith;
      import org.junit.runners.Parameterized;
    @@ -41,6 +43,9 @@ import org.junit.runners.Parameterized.Parameters;
      @RunWith(Parameterized.class)
      public class TestBeelineArgParsing {
        private static final Log LOG = LogFactory.getLog(TestBeelineArgParsing.class.getName());
    +
    + private static final String dummyDriverClazzName = "DummyDriver";
    +
        private String connectionString;
        private String driverClazzName;
        private String driverJarFileName;
    @@ -85,15 +90,17 @@ public class TestBeelineArgParsing {
          }
        }

    - @Parameters
    - public static Collection<Object[]> data() {
    - return Arrays.asList(
    - new Object[][] {
    - {"jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver", "postgresql-9.3.jdbc3.jar",
    - true},
    - {"jdbc:dummy://host:5432/testdb", "org.apache.dummy.DummyDriver",
    - "DummyDriver-1.0-SNAPSHOT.jar", false}
    - });
    + @Parameters public static Collection<Object[]> data() throws IOException, InterruptedException {
    + // generate the dummy driver by using txt file
    + String u = HiveTestUtils.getFileFromClasspath("DummyDriver.txt");
    + File jarFile = HiveTestUtils.genLocalJarForTest(u, dummyDriverClazzName);
    + String pathToDummyDriver = jarFile.getAbsolutePath();
    + return Arrays.asList(new Object[][] {
    + { "jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver",
    + System.getProperty("maven.local.repository") + File.separator + "postgresql"
    + + File.separator + "postgresql" + File.separator + "9.1-901.jdbc4" + File.separator
    + + "postgresql-9.1-901.jdbc4.jar", true },
    + { "jdbc:dummy://host:5432/testdb", dummyDriverClazzName, pathToDummyDriver, false } });
        }

        @Test
    @@ -217,9 +224,8 @@ public class TestBeelineArgParsing {
          Assert.assertNull(bl.findLocalDriver(connectionString));

          LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName);
    - String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName);

    - bl.addLocalJar(mysqlDriverPath);
    + bl.addLocalJar(driverJarFileName);
          bl.addlocaldrivername(driverClazzName);
          Assert.assertEquals(bl.findLocalDriver(connectionString).getClass().getName(), driverClazzName);
        }
    @@ -229,9 +235,7 @@ public class TestBeelineArgParsing {
          TestBeeline bl = new TestBeeline();

          LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName);
    - String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName);
    -
    - bl.addLocalJar(mysqlDriverPath);
    + bl.addLocalJar(driverJarFileName);
          if (!defaultSupported) {
            Assert.assertNull(bl.findLocalDriver(connectionString));
          } else {

    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar
    ----------------------------------------------------------------------
    diff --git a/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar b/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar
    deleted file mode 100644
    index 3dadc9e..0000000
    Binary files a/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar and /dev/null differ

    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/resources/DummyDriver.txt
    ----------------------------------------------------------------------
    diff --git a/beeline/src/test/resources/DummyDriver.txt b/beeline/src/test/resources/DummyDriver.txt
    new file mode 100644
    index 0000000..05f8b12
    --- /dev/null
    +++ b/beeline/src/test/resources/DummyDriver.txt
    @@ -0,0 +1,59 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +import java.sql.Connection;
    +import java.sql.Driver;
    +import java.sql.DriverPropertyInfo;
    +import java.sql.SQLException;
    +import java.sql.SQLFeatureNotSupportedException;
    +import java.util.Properties;
    +import java.util.logging.Logger;
    +
    +public class DummyDriver implements Driver {
    + @Override public Connection connect(String url, Properties info) throws SQLException {
    + return null;
    + }
    +
    + @Override public boolean acceptsURL(String url) throws SQLException {
    + if (url == null) {
    + return false;
    + } else {
    + return url.startsWith("jdbc:dummy://");
    + }
    + }
    +
    + @Override public DriverPropertyInfo[] getPropertyInfo(String url, Properties info)
    + throws SQLException {
    + return new DriverPropertyInfo[0];
    + }
    +
    + @Override public int getMajorVersion() {
    + return 0;
    + }
    +
    + @Override public int getMinorVersion() {
    + return 0;
    + }
    +
    + @Override public boolean jdbcCompliant() {
    + return false;
    + }
    +
    + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
    + return null;
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/resources/postgresql-9.3.jdbc3.jar
    ----------------------------------------------------------------------
    diff --git a/beeline/src/test/resources/postgresql-9.3.jdbc3.jar b/beeline/src/test/resources/postgresql-9.3.jdbc3.jar
    deleted file mode 100644
    index f537b98..0000000
    Binary files a/beeline/src/test/resources/postgresql-9.3.jdbc3.jar and /dev/null differ

    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hive/common/util/HiveTestUtils.java b/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
    index db34494..06caa53 100644
    --- a/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
    +++ b/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
    @@ -18,14 +18,27 @@

      package org.apache.hive.common.util;

    +import java.io.BufferedReader;
    +import java.io.File;
    +import java.io.IOException;
    +import java.io.InputStreamReader;
      import java.net.URL;

    +import com.google.common.io.Files;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.hive.common.classification.InterfaceAudience;
      import org.apache.hadoop.hive.common.classification.InterfaceStability;

      @InterfaceAudience.Private
      @InterfaceStability.Unstable
      public class HiveTestUtils {
    + public static final Log LOG = LogFactory.getLog(HiveTestUtils.class);
    +
    + public final static String JAVA_FILE_EXT = ".java";
    + public final static String CLAZZ_FILE_EXT = ".class";
    + public final static String JAR_FILE_EXT = ".jar";
    + public final static String TXT_FILE_EXT = ".txt";

        public static String getFileFromClasspath(String name) {
          URL url = ClassLoader.getSystemResource(name);
    @@ -34,4 +47,37 @@ public class HiveTestUtils {
          }
          return url.getPath();
        }
    +
    + private static void executeCmd(String[] cmdArr, File dir) throws IOException, InterruptedException {
    + final Process p1 = Runtime.getRuntime().exec(cmdArr, null, dir);
    + new Thread(new Runnable() {
    + public void run() {
    + BufferedReader input = new BufferedReader(new InputStreamReader(p1.getErrorStream()));
    + String line;
    + try {
    + while ((line = input.readLine()) != null) {
    + System.out.println(line);
    + }
    + } catch (IOException e) {
    + LOG.error("Failed to execute the command due the exception " + e);
    + }
    + }
    + }).start();
    + p1.waitFor();
    + }
    +
    + public static File genLocalJarForTest(String pathToClazzFile, String clazzName)
    + throws IOException, InterruptedException {
    + String u = pathToClazzFile;
    + File dir = new File(u);
    + File parentDir = dir.getParentFile();
    + File f = new File(parentDir, clazzName + JAVA_FILE_EXT);
    + Files.copy(dir, f);
    + executeCmd(new String[] { "javac", clazzName + JAVA_FILE_EXT }, parentDir);
    + executeCmd(new String[] { "jar", "cf", clazzName + JAR_FILE_EXT, clazzName + CLAZZ_FILE_EXT },
    + parentDir);
    + f.delete();
    + new File(parentDir, clazzName + CLAZZ_FILE_EXT).delete();
    + return new File(parentDir, clazzName + JAR_FILE_EXT);
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
    index 45ad22a..9e16c0c 100644
    --- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
    @@ -57,15 +57,8 @@ public class TestSessionState {
        private final static String reloadClazzFileName = "RefreshedJarClass.jar";
        private final static String versionMethodName = "version";
        private final static String RELOADED_CLAZZ_PREFIX_NAME = "RefreshedJarClass";
    - private final static String JAVA_FILE_EXT = ".java";
    - private final static String CLAZZ_FILE_EXT = ".class";
    - private final static String JAR_FILE_EXT = ".jar";
    - private final static String TXT_FILE_EXT = ".txt";
        private final static String V1 = "V1";
        private final static String V2 = "V2";
    - private final String clazzFile = RELOADED_CLAZZ_PREFIX_NAME + CLAZZ_FILE_EXT;
    - private final String jarFile = RELOADED_CLAZZ_PREFIX_NAME + JAR_FILE_EXT;
    - private final String javaFile = RELOADED_CLAZZ_PREFIX_NAME + JAVA_FILE_EXT;
        private static String hiveReloadPath;
        private File reloadFolder;
        public static final Log LOG = LogFactory.getLog(TestSessionState.class);
    @@ -191,35 +184,10 @@ public class TestSessionState {
        }

        private void generateRefreshJarFiles(String version) throws IOException, InterruptedException {
    - String u = HiveTestUtils.getFileFromClasspath(
    - RELOADED_CLAZZ_PREFIX_NAME + version + TXT_FILE_EXT);
    - File dir = new File(u);
    - File parentDir = dir.getParentFile();
    - File f = new File(parentDir, javaFile);
    - Files.copy(dir, f);
    - executeCmd(new String[]{"javac", javaFile}, parentDir);
    - executeCmd(new String[]{"jar", "cf", jarFile, clazzFile}, parentDir);
    - Files.move(new File(parentDir, jarFile), new File(parentDir, jarFile + "." + version));
    - f.delete();
    - new File(parentDir, clazzFile).delete();
    - }
    -
    - private void executeCmd(String[] cmdArr, File dir) throws IOException, InterruptedException {
    - final Process p1 = Runtime.getRuntime().exec(cmdArr, null, dir);
    - new Thread(new Runnable() {
    - public void run() {
    - BufferedReader input = new BufferedReader(new InputStreamReader(p1.getErrorStream()));
    - String line;
    - try {
    - while ((line = input.readLine()) != null) {
    - System.out.println(line);
    - }
    - } catch (IOException e) {
    - LOG.error("Failed to execute the command due the exception " + e);
    - }
    - }
    - }).start();
    - p1.waitFor();
    + String u = HiveTestUtils
    + .getFileFromClasspath(RELOADED_CLAZZ_PREFIX_NAME + version + HiveTestUtils.TXT_FILE_EXT);
    + File jarFile = HiveTestUtils.genLocalJarForTest(u, RELOADED_CLAZZ_PREFIX_NAME);
    + Files.move(jarFile, new File(jarFile.getAbsolutePath() + "." + version));
        }

        @Test
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10892 : TestHCatClient should not accept external metastore param from -Dhive.metastore.uris (Sushanth Sowmyan, reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4140da59
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4140da59
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4140da59

    Branch: refs/heads/hbase-metastore
    Commit: 4140da59658d0a7d3b59d827fe0ba582b1e2816e
    Parents: b4f57d8
    Author: Sushanth Sowmyan <khorgath@gmail.com>
    Authored: Wed Jun 3 14:58:22 2015 -0700
    Committer: Sushanth Sowmyan <khorgath@gmail.com>
    Committed: Wed Jun 3 14:59:05 2015 -0700

    ----------------------------------------------------------------------
      .../src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/4140da59/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
    index 5f94b13..8992d552c 100644
    --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
    +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
    @@ -131,7 +131,7 @@ public class TestHCatClient {
        public static void startMetaStoreServer() throws Exception {

          hcatConf = new HiveConf(TestHCatClient.class);
    - String metastoreUri = System.getProperty(HiveConf.ConfVars.METASTOREURIS.varname);
    + String metastoreUri = System.getProperty("test."+HiveConf.ConfVars.METASTOREURIS.varname);
          if (metastoreUri != null) {
            hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metastoreUri);
            useExternalMS = true;
  • Gates at Jun 5, 2015 at 11:16 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/pom.xml
    ----------------------------------------------------------------------
    diff --git a/pom.xml b/pom.xml
    index e60bb35..9b40909 100644
    --- a/pom.xml
    +++ b/pom.xml
    @@ -116,6 +116,7 @@
          <commons-pool.version>1.5.4</commons-pool.version>
          <commons-dbcp.version>1.4</commons-dbcp.version>
          <derby.version>10.11.1.1</derby.version>
    + <dropwizard.version>3.1.0</dropwizard.version>
          <guava.version>14.0.1</guava.version>
          <groovy.version>2.1.6</groovy.version>
          <hadoop-20S.version>1.2.1</hadoop-20S.version>
    @@ -128,6 +129,8 @@
          <httpcomponents.core.version>4.4</httpcomponents.core.version>
          <ivy.version>2.4.0</ivy.version>
          <jackson.version>1.9.2</jackson.version>
    + <!-- jackson 1 and 2 lines can coexist without issue, as they have different artifactIds -->
    + <jackson.new.version>2.4.2</jackson.new.version>
          <javaewah.version>0.3.2</javaewah.version>
          <javolution.version>5.5.1</javolution.version>
          <jdo-api.version>3.0.1</jdo-api.version>

    http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/service/src/java/org/apache/hive/service/server/HiveServer2.java
    ----------------------------------------------------------------------
    diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
    index 58e8e49..7820ed5 100644
    --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
    +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
    @@ -42,14 +42,15 @@ import org.apache.curator.framework.api.CuratorEvent;
      import org.apache.curator.framework.api.CuratorEventType;
      import org.apache.curator.framework.recipes.nodes.PersistentEphemeralNode;
      import org.apache.curator.retry.ExponentialBackoffRetry;
    +import org.apache.hadoop.hive.common.JvmPauseMonitor;
      import org.apache.hadoop.hive.common.LogUtils;
      import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
    +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
      import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
      import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
      import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
    -import org.apache.hadoop.hive.shims.ShimLoader;
      import org.apache.hadoop.hive.shims.Utils;
      import org.apache.hadoop.security.UserGroupInformation;
      import org.apache.hive.common.util.HiveStringUtils;
    @@ -305,6 +306,15 @@ public class HiveServer2 extends CompositeService {
          LOG.info("Shutting down HiveServer2");
          HiveConf hiveConf = this.getHiveConf();
          super.stop();
    + // Shutdown Metrics
    + if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) {
    + try {
    + MetricsFactory.getMetricsInstance().deInit();
    + } catch (Exception e) {
    + LOG.error("error in Metrics deinit: " + e.getClass().getName() + " "
    + + e.getMessage(), e);
    + }
    + }
          // Remove this server instance from ZooKeeper if dynamic service discovery is set
          if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
            try {
    @@ -344,7 +354,18 @@ public class HiveServer2 extends CompositeService {
              server = new HiveServer2();
              server.init(hiveConf);
              server.start();
    - ShimLoader.getHadoopShims().startPauseMonitor(hiveConf);
    +
    + if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) {
    + MetricsFactory.getMetricsInstance().init(hiveConf);
    + }
    + try {
    + JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(hiveConf);
    + pauseMonitor.start();
    + } catch (Throwable t) {
    + LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
    + "warned upon.", t);
    + }
    +
              // If we're supporting dynamic service discovery, we'll add the service uri for this
              // HiveServer2 instance to Zookeeper as a znode.
              if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    ----------------------------------------------------------------------
    diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    index 6d8166c..ffffcb7 100644
    --- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    +++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    @@ -159,11 +159,6 @@ public class Hadoop20SShims extends HadoopShimsSecure {
        }

        @Override
    - public void startPauseMonitor(Configuration conf) {
    - /* no supported */
    - }
    -
    - @Override
        public boolean isLocalMode(Configuration conf) {
          return "local".equals(getJobLauncherRpcAddress(conf));
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    ----------------------------------------------------------------------
    diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    index 19324b8..5ddab98 100644
    --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    @@ -212,19 +212,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
        }

        @Override
    - public void startPauseMonitor(Configuration conf) {
    - try {
    - Class.forName("org.apache.hadoop.util.JvmPauseMonitor");
    - org.apache.hadoop.util.JvmPauseMonitor pauseMonitor = new org.apache.hadoop.util
    - .JvmPauseMonitor(conf);
    - pauseMonitor.start();
    - } catch (Throwable t) {
    - LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
    - "warned upon.", t);
    - }
    - }
    -
    - @Override
        public boolean isLocalMode(Configuration conf) {
          return "local".equals(conf.get("mapreduce.framework.name"));
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    ----------------------------------------------------------------------
    diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    index 5a6bc44..5b7e7f6 100644
    --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    @@ -146,8 +146,6 @@ public interface HadoopShims {

        public JobContext newJobContext(Job job);

    - public void startPauseMonitor(Configuration conf);
    -
        /**
         * Check wether MR is configured to run in local-mode
         * @param conf
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10855: with vectorization enabled join operation involving interval_day_time fails (Matt McCline reviewed by Gunther Hagleitner)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09100831
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09100831
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09100831

    Branch: refs/heads/hbase-metastore
    Commit: 09100831adff7589ee48e735a4beac6ebb25cb3e
    Parents: 24d3307
    Author: Matt McCline <mmccline@hortonworks.com>
    Authored: Tue Jun 2 20:32:14 2015 -0700
    Committer: Matt McCline <mmccline@hortonworks.com>
    Committed: Tue Jun 2 20:32:14 2015 -0700

    ----------------------------------------------------------------------
      data/files/vectortab_a_1k | 1000 ++++++++++++++++++
      data/files/vectortab_b_1k | 1000 ++++++++++++++++++
      .../test/resources/testconfiguration.properties | 1 +
      .../hive/ql/exec/vector/ColumnVector.java | 10 +
      .../ql/exec/vector/VectorColumnSetInfo.java | 42 +-
      .../hive/ql/exec/vector/VectorCopyRow.java | 35 +-
      .../ql/exec/vector/VectorizationContext.java | 46 +
      .../mapjoin/VectorMapJoinCommonOperator.java | 50 +-
      .../clientpositive/vector_interval_mapjoin.q | 87 ++
      .../tez/vector_interval_mapjoin.q.out | 281 +++++
      .../vector_interval_mapjoin.q.out | 281 +++++
      11 files changed, 2797 insertions(+), 36 deletions(-)
    ----------------------------------------------------------------------
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10919: Windows: create table with JsonSerDe failed via beeline unless you add hcatalog core jar to classpath (Hari Sankar Sivarama Subramaniyan, reviewed by Sushanth Sowmyan


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/949ddaae
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/949ddaae
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/949ddaae

    Branch: refs/heads/hbase-metastore
    Commit: 949ddaae125a110d08e9cd6d565b7193e4941a37
    Parents: aec98e9
    Author: Gunther Hagleitner <gunther@apache.org>
    Authored: Wed Jun 3 21:17:01 2015 -0700
    Committer: Gunther Hagleitner <gunther@apache.org>
    Committed: Wed Jun 3 21:17:01 2015 -0700

    ----------------------------------------------------------------------
      bin/hive.cmd | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/949ddaae/bin/hive.cmd
    ----------------------------------------------------------------------
    diff --git a/bin/hive.cmd b/bin/hive.cmd
    index ba7e443..9080796 100644
    --- a/bin/hive.cmd
    +++ b/bin/hive.cmd
    @@ -158,7 +158,7 @@ if not exist %HIVE_LIB%\hive-cli-*.jar (
       exit /b 1
      )

    -set CLASSPATH=%CLASSPATH%;%HIVE_LIB%\*
    +set CLASSPATH=%CLASSPATH%;%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\*;

      @rem maybe we should just make users set HADOOP_HOME env variable as a prereq
      @rem in the next iteration, use "where" command to find directory of hadoop install from path
  • Gates at Jun 5, 2015 at 11:16 pm
    HIVE-10748 Replace StringBuffer with StringBuilder where possible (Alexander Pivovarov, reviewed by Ashutosh Chauhan, Sergio Peña and Swarnim Kulkarni)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aaf3bd5
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aaf3bd5
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aaf3bd5

    Branch: refs/heads/hbase-metastore
    Commit: 8aaf3bd5b7702b4f92503dbf7ffcccad82633d6d
    Parents: 715871a
    Author: Alexander Pivovarov <apivovarov@gmail.com>
    Authored: Wed May 20 19:05:42 2015 -0700
    Committer: Alexander Pivovarov <apivovarov@gmail.com>
    Committed: Tue Jun 2 11:35:51 2015 -0700

    ----------------------------------------------------------------------
      .../common/jsonexplain/tez/TezJsonParser.java | 2 +-
      .../hive/common/type/TestHiveBaseChar.java | 2 +-
      .../hive/common/type/TestHiveVarchar.java | 22 ++++++++++----------
      .../hive/hcatalog/common/HCatException.java | 2 +-
      .../hive/hcatalog/data/TestJsonSerDe.java | 2 +-
      .../streaming/DelimitedInputWriter.java | 2 +-
      .../hive/hcatalog/streaming/HiveEndPoint.java | 2 +-
      .../hive/hcatalog/streaming/TestStreaming.java | 2 +-
      .../apache/hive/jdbc/HivePreparedStatement.java | 2 +-
      .../metastore/txn/CompactionTxnHandler.java | 9 ++++----
      .../apache/hadoop/hive/ql/exec/ExplainTask.java | 4 ++--
      .../apache/hadoop/hive/ql/exec/Utilities.java | 9 ++++----
      .../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
      .../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 4 ++--
      .../ql/exec/vector/VectorizedBatchUtil.java | 2 +-
      .../hive/ql/optimizer/GenMRTableScan1.java | 2 +-
      .../hive/ql/optimizer/GenMapRedUtils.java | 5 ++++-
      .../ql/optimizer/ReduceSinkMapJoinProc.java | 2 +-
      .../ql/optimizer/calcite/RelOptHiveTable.java | 2 +-
      .../spark/SparkReduceSinkMapJoinProc.java | 2 +-
      .../hive/ql/parse/ProcessAnalyzeTable.java | 4 ++--
      .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
      .../parse/spark/SparkProcessAnalyzeTable.java | 2 +-
      .../apache/hadoop/hive/ql/plan/FilterDesc.java | 4 ++--
      .../apache/hadoop/hive/ql/plan/PlanUtils.java | 19 ++++++++++-------
      .../hadoop/hive/ql/plan/TableScanDesc.java | 2 +-
      .../hive/ql/txn/compactor/CompactorMR.java | 9 ++++----
      .../hadoop/hive/ql/txn/compactor/Initiator.java | 2 +-
      .../hadoop/hive/ql/txn/compactor/Worker.java | 3 +--
      .../objectinspector/ObjectInspectorUtils.java | 2 +-
      .../hive/serde2/lazy/TestLazySimpleSerDe.java | 9 +-------
      .../apache/hive/service/auth/HttpAuthUtils.java | 2 +-
      32 files changed, 72 insertions(+), 69 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
    index 6d6bbc2..43ddff3 100644
    --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
    +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
    @@ -125,7 +125,7 @@ public class TezJsonParser implements JsonParser {
            if (candidate.tezStageDependency != null && candidate.tezStageDependency.size() > 0) {
              outputStream.println("Vertex dependency in root stage");
              for (Entry<Vertex, List<Connection>> entry : candidate.tezStageDependency.entrySet()) {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
                sb.append(entry.getKey().name);
                sb.append(" <- ");
                boolean printcomma = false;

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
    ----------------------------------------------------------------------
    diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
    index 012c28b..2848465 100644
    --- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
    +++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
    @@ -55,7 +55,7 @@ public class TestHiveBaseChar extends TestCase {
        }

        public static String createRandomSupplementaryCharString(int len) {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          for (int idx = 0; idx < len; ++idx) {
            sb.appendCodePoint(getRandomCodePoint(' '));
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
    ----------------------------------------------------------------------
    diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
    index 309d042..44e2338 100644
    --- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
    +++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
    @@ -18,19 +18,19 @@
      package org.apache.hadoop.hive.common.type;


    -import org.apache.hadoop.hive.common.type.HiveVarchar;
    -import org.apache.hadoop.hive.common.LogUtils;
    -import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
    +import static org.junit.Assert.assertEquals;
    +import static org.junit.Assert.assertFalse;
    +import static org.junit.Assert.assertTrue;

    -import java.io.BufferedReader;
    -import java.io.IOException;
    -import java.io.InputStreamReader;
      import java.util.Random;

    -import com.google.code.tempusfugit.concurrency.annotations.*;
    -import com.google.code.tempusfugit.concurrency.*;
    -import org.junit.*;
    -import static org.junit.Assert.*;
    +import org.junit.Rule;
    +import org.junit.Test;
    +
    +import com.google.code.tempusfugit.concurrency.ConcurrentRule;
    +import com.google.code.tempusfugit.concurrency.RepeatingRule;
    +import com.google.code.tempusfugit.concurrency.annotations.Concurrent;
    +import com.google.code.tempusfugit.concurrency.annotations.Repeating;

      public class TestHiveVarchar {
        @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
    @@ -80,7 +80,7 @@ public class TestHiveVarchar {
          // Try with supplementary characters
          for (int idx1 = 0; idx1 < lengths.length; ++idx1) {
            // Create random test string
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
            int curLen = lengths[idx1];
            for (int idx2 = 0; idx2 < curLen; ++idx2) {
              sb.appendCodePoint(getRandomCodePoint(' '));

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
    index 265d08d..3749da6 100644
    --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
    +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
    @@ -104,7 +104,7 @@ public class HCatException extends IOException {
        public static String buildErrorMessage(ErrorType type, String extraMessage, Throwable cause) {

          //Initial message is just the error type message
    - StringBuffer message = new StringBuffer(HCatException.class.getName());
    + StringBuilder message = new StringBuilder(HCatException.class.getName());
          message.append(" : " + type.getErrorCode());
          message.append(" : " + type.getErrorMessage());


    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
    index 2947c43..618f39b 100644
    --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
    +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
    @@ -227,7 +227,7 @@ public class TestJsonSerDe extends TestCase {
            return "";
          }

    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          int numStrings = columnNames.split(",").length;
          sb.append("_col0");
          for (int i = 1; i < numStrings; i++) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
    index eae91cb..6dc69f0 100644
    --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
    +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
    @@ -186,7 +186,7 @@ public class DelimitedInputWriter extends AbstractRecordWriter {
        // handles nulls in items[]
        // TODO: perhaps can be made more efficient by creating a byte[] directly
        private static byte[] join(String[] items, char separator) {
    - StringBuffer buff = new StringBuffer(100);
    + StringBuilder buff = new StringBuilder(100);
          if(items.length == 0)
            return "".getBytes();
          int i=0;

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
    index 3c25486..7e99008 100644
    --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
    +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
    @@ -418,7 +418,7 @@ public class HiveEndPoint {
              throw new IllegalArgumentException("Partition values:" + partVals +
                      ", does not match the partition Keys in table :" + partKeys );
            }
    - StringBuffer buff = new StringBuffer(partKeys.size()*20);
    + StringBuilder buff = new StringBuilder(partKeys.size()*20);
            buff.append(" ( ");
            int i=0;
            for (FieldSchema schema : partKeys) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
    index 329e5da..c0af533 100644
    --- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
    +++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
    @@ -794,7 +794,7 @@ public class TestStreaming {
            throw new IllegalArgumentException("Partition values:" + partVals
                    + ", does not match the partition Keys in table :" + partKeys );
          }
    - StringBuffer buff = new StringBuffer(partKeys.size()*20);
    + StringBuilder buff = new StringBuilder(partKeys.size()*20);
          buff.append(" ( ");
          int i=0;
          for (FieldSchema schema : partKeys) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
    ----------------------------------------------------------------------
    diff --git a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
    index 8a0671f..2625155 100644
    --- a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
    +++ b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
    @@ -132,7 +132,7 @@ public class HivePreparedStatement extends HiveStatement implements PreparedStat
            return sql;
          }

    - StringBuffer newSql = new StringBuffer(sql);
    + StringBuilder newSql = new StringBuilder(sql);

          int paramLoc = 1;
          while (getCharIndexFromSqlByParamLocation(sql, '?', paramLoc) > 0) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
    index 52147bc..26e72be 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
    @@ -340,7 +340,7 @@ public class CompactionTxnHandler extends TxnHandler {
              if (txnids.size() > 0) {

                // Remove entries from txn_components, as there may be aborted txn components
    - StringBuffer buf = new StringBuffer();
    + StringBuilder buf = new StringBuilder();
                buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
                boolean first = true;
                for (long id : txnids) {
    @@ -404,7 +404,7 @@ public class CompactionTxnHandler extends TxnHandler {
              Set<Long> txnids = new HashSet<Long>();
              while (rs.next()) txnids.add(rs.getLong(1));
              if (txnids.size() > 0) {
    - StringBuffer buf = new StringBuffer("delete from TXNS where txn_id in (");
    + StringBuilder buf = new StringBuilder("delete from TXNS where txn_id in (");
                boolean first = true;
                for (long tid : txnids) {
                  if (first) first = false;
    @@ -412,8 +412,9 @@ public class CompactionTxnHandler extends TxnHandler {
                  buf.append(tid);
                }
                buf.append(")");
    - LOG.debug("Going to execute update <" + buf.toString() + ">");
    - int rc = stmt.executeUpdate(buf.toString());
    + String bufStr = buf.toString();
    + LOG.debug("Going to execute update <" + bufStr + ">");
    + int rc = stmt.executeUpdate(bufStr);
                LOG.debug("Removed " + rc + " records from txns");
                LOG.debug("Going to commit");
                dbConn.commit();

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    index ada79bd..35c4cfc 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    @@ -854,7 +854,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
            }
          }
          else {
    - StringBuffer s = new StringBuffer();
    + StringBuilder s = new StringBuilder();
            first = true;
            for (Task<?> parent : task.getParentTasks()) {
              if (!first) {
    @@ -886,7 +886,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {

          if (task instanceof ConditionalTask
              && ((ConditionalTask) task).getListTasks() != null) {
    - StringBuffer s = new StringBuffer();
    + StringBuilder s = new StringBuilder();
            first = true;
            for (Task<?> con : ((ConditionalTask) task).getListTasks()) {
              if (!first) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    index 5d58839..5f92d11 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    @@ -1811,11 +1811,12 @@ public final class Utilities {
        private static String adjustBucketNumLen(String bucketNum, String taskId) {
          int bucketNumLen = bucketNum.length();
          int taskIdLen = taskId.length();
    - StringBuffer s = new StringBuffer();
    + StringBuilder s = new StringBuilder();
          for (int i = 0; i < taskIdLen - bucketNumLen; i++) {
            s.append("0");
          }
    - return s.toString() + bucketNum;
    + s.append(bucketNum);
    + return s.toString();
        }

        /**
    @@ -1836,7 +1837,7 @@ public final class Utilities {
            return filename.replaceAll(oldTaskId, newTaskId);
          }

    - StringBuffer snew = new StringBuffer();
    + StringBuilder snew = new StringBuilder();
          for (int idx = 0; idx < spl.length - 1; idx++) {
            if (idx > 0) {
              snew.append(oldTaskId);
    @@ -3169,7 +3170,7 @@ public final class Utilities {
         * @return a string with escaped '_' and '%'.
         */
        public static String escapeSqlLike(String key) {
    - StringBuffer sb = new StringBuffer(key.length());
    + StringBuilder sb = new StringBuilder(key.length());
          for (char c: key.toCharArray()) {
            switch(c) {
            case '_':

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
    index a9d2dbf..3fceeb0 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
    @@ -62,7 +62,7 @@ abstract class SparkJobMonitor {
            return;
          }

    - StringBuffer reportBuffer = new StringBuffer();
    + StringBuilder reportBuffer = new StringBuilder();
          SimpleDateFormat dt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
          String currentDate = dt.format(new Date());
          reportBuffer.append(currentDate + "\t");

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    index 78caba8..aa84850 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
    @@ -596,7 +596,7 @@ public class TezJobMonitor {

        private void printStatusInPlace(Map<String, Progress> progressMap, long startTime,
            boolean vextexStatusFromAM, DAGClient dagClient) {
    - StringBuffer reportBuffer = new StringBuffer();
    + StringBuilder reportBuffer = new StringBuilder();
          int sumComplete = 0;
          int sumTotal = 0;

    @@ -781,7 +781,7 @@ public class TezJobMonitor {
        }

        private String getReport(Map<String, Progress> progressMap) {
    - StringBuffer reportBuffer = new StringBuffer();
    + StringBuilder reportBuffer = new StringBuilder();

          SortedSet<String> keys = new TreeSet<String>(progressMap.keySet());
          for (String s: keys) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
    index 4a16b4c..99cb620 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
    @@ -643,7 +643,7 @@ public class VectorizedBatchUtil {
        }

        public static void debugDisplayOneRow(VectorizedRowBatch batch, int index, String prefix) {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          sb.append(prefix + " row " + index + " ");
          for (int column = 0; column < batch.cols.length; column++) {
            ColumnVector colVector = batch.cols[column];

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
    index c5f03d9..eed1d7c 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
    @@ -187,7 +187,7 @@ public class GenMRTableScan1 implements NodeProcessor {
            ParseContext parseCtx, Task<? extends Serializable> currTask,
            StatsWork statsWork, Task<StatsWork> statsTask) throws SemanticException {
          String aggregationKey = op.getConf().getStatsAggPrefix();
    - StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
    + StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
          List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(op, aggregationKeyBuffer);
          aggregationKey = aggregationKeyBuffer.toString();


    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    index acd9bf5..29854d8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    @@ -18,6 +18,7 @@

      package org.apache.hadoop.hive.ql.optimizer;

    +import java.io.IOException;
      import java.io.Serializable;
      import java.util.ArrayList;
      import java.util.Collections;
    @@ -1814,7 +1815,7 @@ public final class GenMapRedUtils {
        }

        public static List<Path> getInputPathsForPartialScan(TableScanOperator tableScanOp,
    - StringBuffer aggregationKey) throws SemanticException {
    + Appendable aggregationKey) throws SemanticException {
          List<Path> inputPaths = new ArrayList<Path>();
          switch (tableScanOp.getConf().getTableMetadata().getTableSpec().specType) {
            case TABLE_ONLY:
    @@ -1829,6 +1830,8 @@ public final class GenMapRedUtils {
              } catch (MetaException e) {
                throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AGGKEY.getMsg(
                    part.getDataLocation().toString() + e.getMessage()));
    + } catch (IOException e) {
    + throw new RuntimeException(e);
              }
              inputPaths.add(part.getDataLocation());
              break;

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
    index f7e1dbc..bca91dd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
    @@ -283,7 +283,7 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {

          Map<Byte, List<ExprNodeDesc>> keyExprMap = mapJoinOp.getConf().getKeys();
          List<ExprNodeDesc> keyCols = keyExprMap.get(Byte.valueOf((byte) 0));
    - StringBuffer keyOrder = new StringBuffer();
    + StringBuilder keyOrder = new StringBuilder();
          for (ExprNodeDesc k: keyCols) {
            keyOrder.append("+");
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
    index 43882e7..6c0bd25 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
    @@ -221,7 +221,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
        }

        private String getColNamesForLogging(Set<String> colLst) {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          boolean firstEntry = true;
          for (String colName : colLst) {
            if (firstEntry) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
    index e477f04..fd42959 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
    @@ -207,7 +207,7 @@ public class SparkReduceSinkMapJoinProc implements NodeProcessor {

          Map<Byte, List<ExprNodeDesc>> keyExprMap = mapJoinOp.getConf().getKeys();
          List<ExprNodeDesc> keyCols = keyExprMap.get(Byte.valueOf((byte) 0));
    - StringBuffer keyOrder = new StringBuffer();
    + StringBuilder keyOrder = new StringBuilder();
          for (int i = 0; i < keyCols.size(); i++) {
            keyOrder.append("+");
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
    index 7108a47..f8d6905 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
    @@ -168,12 +168,12 @@ public class ProcessAnalyzeTable implements NodeProcessor {
         *
         * It is composed of PartialScanTask followed by StatsTask.
         */
    - private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
    + private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
            StatsWork statsWork, GenTezProcContext context, Task<StatsWork> statsTask)
                    throws SemanticException {

          String aggregationKey = tableScan.getConf().getStatsAggPrefix();
    - StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
    + StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
          List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
          aggregationKey = aggregationKeyBuffer.toString();


    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    index d609732..428ba38 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    @@ -955,7 +955,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         *
         */
        private ASTNode findCTEFromName(QB qb, String cteName) {
    - StringBuffer qId = new StringBuffer();
    + StringBuilder qId = new StringBuilder();
          if (qb.getId() != null) {
            qId.append(qb.getId());
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
    index 9e19733..66e148f 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
    @@ -168,7 +168,7 @@ public class SparkProcessAnalyzeTable implements NodeProcessor {
            StatsWork statsWork, GenSparkProcContext context, Task<StatsWork> statsTask)
                    throws SemanticException {
          String aggregationKey = tableScan.getConf().getStatsAggPrefix();
    - StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
    + StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
          List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
          aggregationKey = aggregationKeyBuffer.toString();


    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
    index 8dff2fc..3a1a4af 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
    @@ -104,11 +104,11 @@ public class FilterDesc extends AbstractOperatorDesc {

        @Explain(displayName = "predicate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
        public String getPredicateString() {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          PlanUtils.addExprToStringBuffer(predicate, sb);
          return sb.toString();
        }
    -
    +
        public org.apache.hadoop.hive.ql.plan.ExprNodeDesc getPredicate() {
          return predicate;
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
    index cb0b680..76926e7 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
    @@ -18,6 +18,7 @@

      package org.apache.hadoop.hive.ql.plan;

    +import java.io.IOException;
      import java.util.ArrayList;
      import java.util.Collection;
      import java.util.Collections;
    @@ -413,7 +414,7 @@ public final class PlanUtils {
            // We basically use ReduceSinkOperators and set the transfer to
            // be broadcast (instead of partitioned). As a consequence we use
            // a different SerDe than in the MR mapjoin case.
    - StringBuffer order = new StringBuffer();
    + StringBuilder order = new StringBuilder();
            for (FieldSchema f: fieldSchemas) {
              order.append("+");
            }
    @@ -930,7 +931,7 @@ public final class PlanUtils {
        }

        public static String getExprListString(Collection<? extends ExprNodeDesc> exprs) {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          boolean first = true;
          for (ExprNodeDesc expr: exprs) {
            if (!first) {
    @@ -944,11 +945,15 @@ public final class PlanUtils {
          return sb.length() == 0 ? null : sb.toString();
        }

    - public static void addExprToStringBuffer(ExprNodeDesc expr, StringBuffer sb) {
    - sb.append(expr.getExprString());
    - sb.append(" (type: ");
    - sb.append(expr.getTypeString());
    - sb.append(")");
    + public static void addExprToStringBuffer(ExprNodeDesc expr, Appendable sb) {
    + try {
    + sb.append(expr.getExprString());
    + sb.append(" (type: ");
    + sb.append(expr.getTypeString());
    + sb.append(")");
    + } catch (IOException e) {
    + throw new RuntimeException(e);
    + }
        }

        public static void addInputsForView(ParseContext parseCtx) throws HiveException {

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
    index aa291b9..6282380 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
    @@ -134,7 +134,7 @@ public class TableScanDesc extends AbstractOperatorDesc {

        @Explain(displayName = "filterExpr")
        public String getFilterExprString() {
    - StringBuffer sb = new StringBuffer();
    + StringBuilder sb = new StringBuilder();
          PlanUtils.addExprToStringBuffer(filterExpr, sb);
          return sb.toString();
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    index b8e18ea..c5f2d4d 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    @@ -585,7 +585,7 @@ public class CompactorMR {

          @Override
          public String toString() {
    - StringBuffer buf = new StringBuffer();
    + StringBuilder buf = new StringBuilder();
            buf.append(size());
            buf.append(':');
            if (size() > 0) {
    @@ -631,14 +631,15 @@ public class CompactorMR {

          @Override
          public String toString() {
    - StringBuffer buf = new StringBuffer();
    + StringBuilder buf = new StringBuilder();
            buf.append(size());
            buf.append(':');
            if (size() > 0) {
              for (Path p : this) {
    - buf.append(p.toString().length());
    + String pStr = p.toString();
    + buf.append(pStr.length());
                buf.append(':');
    - buf.append(p.toString());
    + buf.append(pStr);
              }
            }
            return buf.toString();

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    index 847d751..32a9ef8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    @@ -259,7 +259,7 @@ public class Initiator extends CompactorThread {
                HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_PCT_THRESHOLD);
            boolean bigEnough = (float)deltaSize/(float)baseSize > deltaPctThreshold;
            if (LOG.isDebugEnabled()) {
    - StringBuffer msg = new StringBuffer("delta size: ");
    + StringBuilder msg = new StringBuilder("delta size: ");
              msg.append(deltaSize);
              msg.append(" base size: ");
              msg.append(baseSize);

    http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
    index f26225a..e164661 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
    @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
      import org.apache.hadoop.hive.metastore.api.Table;
      import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
      import org.apache.hadoop.hive.metastore.txn.CompactionTxnHandler;
    -import org.apache.hadoop.hive.metastore.txn.TxnHandler;