FAQ
Author: szehon
Date: Sat Aug 9 01:34:24 2014
New Revision: 1616913

URL: http://svn.apache.org/r1616913
Log:
HIVE-7519 : Refactor QTestUtil to remove its duplication with QFileClient for qtest setup and teardown (Ashish via Szehon)

Modified:
     hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
     hive/trunk/data/conf/hive-site.xml
     hive/trunk/data/conf/tez/hive-site.xml
     hive/trunk/data/scripts/q_test_cleanup.sql
     hive/trunk/data/scripts/q_test_init.sql
     hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
     hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
     hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
     hive/trunk/itests/qtest/pom.xml
     hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
     hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
     hive/trunk/ql/src/test/templates/TestCliDriver.vm
     hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm
     hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm
     hive/trunk/ql/src/test/templates/TestParse.vm
     hive/trunk/ql/src/test/templates/TestParseNegative.vm

Modified: hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Sat Aug 9 01:34:24 2014
@@ -141,6 +141,10 @@ public class QTestGenTask extends Task {

    private String hadoopVersion;

+ private String initScript;
+
+ private String cleanupScript;
+
    public void setHadoopVersion(String ver) {
      this.hadoopVersion = ver;
    }
@@ -197,6 +201,22 @@ public class QTestGenTask extends Task {
      return template;
    }

+ public String getInitScript() {
+ return initScript;
+ }
+
+ public void setInitScript(String initScript) {
+ this.initScript = initScript;
+ }
+
+ public String getCleanupScript() {
+ return cleanupScript;
+ }
+
+ public void setCleanupScript(String cleanupScript) {
+ this.cleanupScript = cleanupScript;
+ }
+
    public void setHiveRootDirectory(File hiveRootDirectory) {
      try {
        this.hiveRootDirectory = hiveRootDirectory.getCanonicalPath();
@@ -444,6 +464,8 @@ public class QTestGenTask extends Task {
        ctx.put("clusterMode", clusterMode);
        ctx.put("hiveConfDir", escapePath(hiveConfDir));
        ctx.put("hadoopVersion", hadoopVersion);
+ ctx.put("initScript", initScript);
+ ctx.put("cleanupScript", cleanupScript);

        File outFile = new File(outDir, className + ".java");
        FileWriter writer = new FileWriter(outFile);

Modified: hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hive-site.xml?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/data/conf/hive-site.xml (original)
+++ hive/trunk/data/conf/hive-site.xml Sat Aug 9 01:34:24 2014
@@ -112,6 +112,12 @@
  </property>

  <property>
+ <name>test.data.scripts</name>
+ <value>${hive.root}/data/scripts</value>
+ <description></description>
+</property>
+
+<property>
    <name>hive.jar.path</name>
    <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
    <description></description>

Modified: hive/trunk/data/conf/tez/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/tez/hive-site.xml?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
Binary files - no diff available.

Modified: hive/trunk/data/scripts/q_test_cleanup.sql
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/q_test_cleanup.sql?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/data/scripts/q_test_cleanup.sql (original)
+++ hive/trunk/data/scripts/q_test_cleanup.sql Sat Aug 9 01:34:24 2014
@@ -7,4 +7,12 @@ DROP TABLE IF EXISTS srcbucket;
  DROP TABLE IF EXISTS srcbucket2;
  DROP TABLE IF EXISTS srcpart;
  DROP TABLE IF EXISTS primitives;
-
+DROP TABLE IF EXISTS dest1;
+DROP TABLE IF EXISTS dest2;
+DROP TABLE IF EXISTS dest3;
+DROP TABLE IF EXISTS dest4;
+DROP TABLE IF EXISTS dest4_sequencefile;
+DROP TABLE IF EXISTS dest_j1;
+DROP TABLE IF EXISTS dest_g1;
+DROP TABLE IF EXISTS dest_g2;
+DROP TABLE IF EXISTS fetchtask_ioexception;

Modified: hive/trunk/data/scripts/q_test_init.sql
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/q_test_init.sql?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/data/scripts/q_test_init.sql (original)
+++ hive/trunk/data/scripts/q_test_init.sql Sat Aug 9 01:34:24 2014
@@ -3,7 +3,7 @@
  --
  DROP TABLE IF EXISTS src;

-CREATE TABLE src (key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE src (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;

  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE src;

@@ -12,7 +12,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
  --
  DROP TABLE IF EXISTS src1;

-CREATE TABLE src1 (key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE src1 (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;

  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv3.txt" INTO TABLE src1;

@@ -21,7 +21,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
  --
  DROP TABLE IF EXISTS src_json;

-CREATE TABLE src_json (json STRING) STORED AS TEXTFILE;
+CREATE TABLE src_json (json STRING COMMENT 'default') STORED AS TEXTFILE;

  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/json.txt" INTO TABLE src_json;

@@ -31,7 +31,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
  --
  DROP TABLE IF EXISTS src_sequencefile;

-CREATE TABLE src_sequencefile (key STRING, value STRING) STORED AS SEQUENCEFILE;
+CREATE TABLE src_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS SEQUENCEFILE;

  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.seq" INTO TABLE src_sequencefile;

@@ -45,7 +45,7 @@ CREATE TABLE src_thrift
  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer'
  WITH SERDEPROPERTIES (
    'serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex',
- 'serialization.format' = 'com.facebook.thrift.protocol.TBinaryProtocol')
+ 'serialization.format' = 'org.apache.thrift.protocol.TBinaryProtocol')
  STORED AS SEQUENCEFILE;

  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/complex.seq" INTO TABLE src_thrift;
@@ -75,6 +75,8 @@ STORED AS TEXTFILE;

  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket20.txt" INTO TABLE srcbucket2;
  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket21.txt" INTO TABLE srcbucket2;
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket22.txt" INTO TABLE srcbucket2;
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket23.txt" INTO TABLE srcbucket2;


  --
@@ -82,7 +84,7 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
  --
  DROP TABLE IF EXISTS srcpart;

-CREATE TABLE srcpart (key STRING, value STRING)
+CREATE TABLE srcpart (key STRING COMMENT 'default', value STRING COMMENT 'default')
  PARTITIONED BY (ds STRING, hr STRING)
  STORED AS TEXTFILE;

@@ -99,20 +101,46 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.
  OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-09", hr="12");


+--
+-- Table alltypesorc
+--
+DROP TABLE IF EXISTS alltypesorc;
+CREATE TABLE alltypesorc(
+ ctinyint TINYINT,
+ csmallint SMALLINT,
+ cint INT,
+ cbigint BIGINT,
+ cfloat FLOAT,
+ cdouble DOUBLE,
+ cstring1 STRING,
+ cstring2 STRING,
+ ctimestamp1 TIMESTAMP,
+ ctimestamp2 TIMESTAMP,
+ cboolean1 BOOLEAN,
+ cboolean2 BOOLEAN)
+ STORED AS ORC;
+
+LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc"
+OVERWRITE INTO TABLE alltypesorc;
+
+
+--
+-- Table primitives
+--
  DROP TABLE IF EXISTS primitives;
  CREATE TABLE primitives (
- id INT,
- bool_col BOOLEAN,
- tinyint_col TINYINT,
- smallint_col SMALLINT,
- int_col INT,
- bigint_col BIGINT,
- float_col FLOAT,
- double_col DOUBLE,
- date_string_col STRING,
- string_col STRING,
- timestamp_col TIMESTAMP)
-PARTITIONED BY (year INT, month INT)
+ id INT COMMENT 'default',
+ bool_col BOOLEAN COMMENT 'default',
+ tinyint_col TINYINT COMMENT 'default',
+ smallint_col SMALLINT COMMENT 'default',
+ int_col INT COMMENT 'default',
+ bigint_col BIGINT COMMENT 'default',
+ float_col FLOAT COMMENT 'default',
+ double_col DOUBLE COMMENT 'default',
+ date_string_col STRING COMMENT 'default',
+ string_col STRING COMMENT 'default',
+ timestamp_col TIMESTAMP COMMENT 'default')
+PARTITIONED BY (year INT COMMENT 'default', month INT COMMENT 'default')
  ROW FORMAT DELIMITED
    FIELDS TERMINATED BY ','
    ESCAPED BY '\\'
@@ -130,3 +158,60 @@ OVERWRITE INTO TABLE primitives PARTITIO
  LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090401.txt"
  OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=4);

+--
+-- Function qtest_get_java_boolean
+--
+DROP FUNCTION IF EXISTS qtest_get_java_boolean;
+CREATE FUNCTION qtest_get_java_boolean AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean';
+
+--
+-- Table dest1
+--
+DROP TABLE IF EXISTS dest1;
+
+CREATE TABLE dest1 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest2
+--
+DROP TABLE IF EXISTS dest2;
+
+CREATE TABLE dest2 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest3
+--
+DROP TABLE IF EXISTS dest3;
+
+CREATE TABLE dest3 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+PARTITIONED BY (ds STRING, hr STRING)
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+ALTER TABLE dest3 ADD PARTITION (ds='2008-04-08',hr='12');
+
+--
+-- Table dest4
+--
+DROP TABLE IF EXISTS dest4;
+
+CREATE TABLE dest4 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+--
+-- Table dest4_sequencefile
+--
+DROP TABLE IF EXISTS dest4_sequencefile;
+
+CREATE TABLE dest4_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileOutputFormat';
\ No newline at end of file

Modified: hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm (original)
+++ hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -44,9 +44,12 @@ public class $className extends TestCase
    protected void setUp() {

      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";

      try {
- qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
+ qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ setup, initScript, cleanupScript);
      } catch (Exception e) {
        System.err.println("Exception: " + e.getMessage());
        e.printStackTrace();

Modified: hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm (original)
+++ hive/trunk/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -45,9 +45,12 @@ public class $className extends TestCase
    protected void setUp() {

      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";

      try {
- qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
+ qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ setup, initScript, cleanupScript);
      } catch (Exception e) {
        System.err.println("Exception: " + e.getMessage());
        e.printStackTrace();

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java Sat Aug 9 01:34:24 2014
@@ -88,7 +88,7 @@ public class TestLocationQueries extends
          String hadoopVer, String locationSubdir)
        throws Exception
      {
- super(outDir, logDir, miniMr, hadoopVer);
+ super(outDir, logDir, miniMr, hadoopVer, "", "");
        this.locationSubdir = locationSubdir;
      }
    }

Modified: hive/trunk/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/pom.xml?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/qtest/pom.xml (original)
+++ hive/trunk/itests/qtest/pom.xml Sat Aug 9 01:34:24 2014
@@ -420,7 +420,9 @@
                    resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/compiler/" className="TestParse"
                    logFile="${project.build.directory}/testparsegen.log"
                    hadoopVersion="${active.hadoop.version}"
- logDirectory="${project.build.directory}/qfile-results/positive/"/>
+ logDirectory="${project.build.directory}/qfile-results/positive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- Negative Parse -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -433,7 +435,9 @@
                    resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/compiler/errors/" className="TestParseNegative"
                    logFile="${project.build.directory}/testparseneggen.log"
                    hadoopVersion="${active.hadoop.version}"
- logDirectory="${project.build.directory}/qfile-results/negative/"/>
+ logDirectory="${project.build.directory}/qfile-results/negative/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- Cli -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -448,7 +452,9 @@
                    resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestCliDriver"
                    logFile="${project.build.directory}/testclidrivergen.log"
                    logDirectory="${project.build.directory}/qfile-results/clientpositive/"
- hadoopVersion="${active.hadoop.version}"/>
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- Negative Cli -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -463,7 +469,9 @@
                    resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientnegative/" className="TestNegativeCliDriver"
                    logFile="${project.build.directory}/testnegativeclidrivergen.log"
                    logDirectory="${project.build.directory}/qfile-results/clientnegative/"
- hadoopVersion="${active.hadoop.version}"/>
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- Compare Cli -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -477,7 +485,9 @@
                    className="TestCompareCliDriver"
                    logFile="${project.build.directory}/testcompareclidrivergen.log"
                    logDirectory="${project.build.directory}/qfile-results/clientcompare/"
- hadoopVersion="${active.hadoop.version}"/>
+ hadoopVersion="${active.hadoop.version}"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- Minimr -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -493,7 +503,8 @@
                    logFile="${project.build.directory}/testminimrclidrivergen.log"
                    logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                    hadoopVersion="${active.hadoop.version}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <if>
                    <equals arg1="${active.hadoop.version}" arg2="${hadoop-23.version}"/>
@@ -513,7 +524,8 @@
                                logFile="${project.build.directory}/testminitezclidrivergen.log"
                                logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                                hadoopVersion="${active.hadoop.version}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
                    </then>
                    <else>
                    </else>
@@ -533,7 +545,8 @@
                    logFile="${project.build.directory}/testnegativeminimrclidrivergen.log"
                    logDirectory="${project.build.directory}/qfile-results/clientnegative/"
                    hadoopVersion="${hadoopVersion}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- HBase Positive -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -545,7 +558,9 @@
                    clusterMode="${clustermode}"
                    resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseCliDriver"
                    logFile="${project.build.directory}/testhbaseclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"/>
+ logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- HBase Minimr -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -557,7 +572,9 @@
                    clusterMode="miniMR"
                    resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseMinimrCliDriver"
                    logFile="${project.build.directory}/testhbaseminimrclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"/>
+ logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <!-- HBase Negative -->
                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -569,7 +586,9 @@
                    clusterMode="${clustermode}"
                    resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/negative/" className="TestHBaseNegativeCliDriver"
                    logFile="${project.build.directory}/testhbasenegativeclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"/>
+ logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>


                  <!-- Beeline -->
@@ -588,7 +607,7 @@
                          resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestBeeLineDriver"
                          logFile="${project.build.directory}/testbeelinedrivergen.log"
                          logDirectory="${project.build.directory}/qfile-results/beelinepositive/"
- hadoopVersion="${hadoopVersion}" />
+ hadoopVersion="${hadoopVersion}"/>
                      </then>
                  </if>

@@ -606,7 +625,8 @@
                    logFile="${project.build.directory}/testcontribclidrivergen.log"
                    logDirectory="${project.build.directory}/qfile-results/contribclientpositive"
                    hadoopVersion="${hadoopVersion}"
- />
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>

                  <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
                    outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli"
@@ -617,7 +637,9 @@
                    runDisabled="${run_disabled}"
                    resultsDirectory="${basedir}/${hive.path.to.root}/contrib/src/test/results/clientnegative/" className="TestContribNegativeCliDriver"
                    logFile="${project.build.directory}/testcontribnegclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/contribclientnegative"/>
+ logDirectory="${project.build.directory}/qfile-results/contribclientnegative"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>


                </target>

Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java Sat Aug 9 01:34:24 2014
@@ -40,10 +40,11 @@ public class HBaseQTestUtil extends QTes
    private final HConnection conn;

    public HBaseQTestUtil(
- String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup)
+ String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup,
+ String initScript, String cleanupScript)
      throws Exception {

- super(outDir, logDir, miniMr, null);
+ super(outDir, logDir, miniMr, null, initScript, cleanupScript);
      setup.preTest(conf);
      this.conn = setup.getConnection();
      super.init();

Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Sat Aug 9 01:34:24 2014
@@ -38,7 +38,6 @@ import java.io.OutputStreamWriter;
  import java.io.PrintStream;
  import java.io.Serializable;
  import java.io.StringWriter;
-import java.io.UnsupportedEncodingException;
  import java.net.URL;
  import java.util.ArrayList;
  import java.util.Arrays;
@@ -115,6 +114,8 @@ public class QTestUtil {

    public static final String UTF_8 = "UTF-8";
    private static final Log LOG = LogFactory.getLog("QTestUtil");
+ private final String defaultInitScript = "q_test_init.sql";
+ private final String defaultCleanupScript = "q_test_cleanup.sql";

    private String testWarehouse;
    private final String testFiles;
@@ -142,6 +143,10 @@ public class QTestUtil {
    private boolean miniMr = false;
    private String hadoopVer = null;
    private QTestSetup setup = null;
+ private boolean isSessionStateStarted = false;
+
+ private String initScript;
+ private String cleanupScript;

    static {
      for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
@@ -225,8 +230,9 @@ public class QTestUtil {
      }
    }

- public QTestUtil(String outDir, String logDir) throws Exception {
- this(outDir, logDir, MiniClusterType.none, null, "0.20");
+ public QTestUtil(String outDir, String logDir, String initScript, String cleanupScript) throws
+ Exception {
+ this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, cleanupScript);
    }

    public String getOutputDirectory() {
@@ -297,13 +303,14 @@ public class QTestUtil {
      }
    }

- public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer)
+ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer,
+ String initScript, String cleanupScript)
      throws Exception {
- this(outDir, logDir, clusterType, null, hadoopVer);
+ this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript);
    }

    public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
- String confDir, String hadoopVer)
+ String confDir, String hadoopVer, String initScript, String cleanupScript)
      throws Exception {
      this.outDir = outDir;
      this.logDir = logDir;
@@ -354,6 +361,20 @@ public class QTestUtil {

      testFiles = dataDir;

+ // Use the current directory if it is not specified
+ String scriptsDir = conf.get("test.data.scripts");
+ if (scriptsDir == null) {
+ scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
+ }
+ if (initScript.isEmpty()) {
+ initScript = defaultInitScript;
+ }
+ if (cleanupScript.isEmpty()) {
+ cleanupScript = defaultCleanupScript;
+ }
+ this.initScript = scriptsDir + "/" + initScript;
+ this.cleanupScript = scriptsDir + "/" + cleanupScript;
+
      overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));

      setup = new QTestSetup();
@@ -593,14 +614,15 @@ public class QTestUtil {
    }

    public void cleanUp() throws Exception {
- // Drop any tables that remain due to unsuccessful runs
- for (String s : new String[] {"src", "src1", "src_json", "src_thrift",
- "src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1",
- "dest2", "dest3", "dest4", "dest4_sequencefile", "dest_j1", "dest_j2",
- "dest_g1", "dest_g2", "fetchtask_ioexception",
- AllVectorTypesRecord.TABLE_NAME}) {
- db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
+ if(!isSessionStateStarted) {
+ startSessionState();
      }
+ String cleanupCommands = readEntireFileIntoString(new File(cleanupScript));
+ LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ cliDriver.processLine(cleanupCommands);

      // delete any contents in the warehouse dir
      Path p = new Path(testWarehouse);
@@ -653,119 +675,20 @@ public class QTestUtil {
    }

    public void createSources() throws Exception {
-
- startSessionState();
+ if(!isSessionStateStarted) {
+ startSessionState();
+ }
      conf.setBoolean("hive.test.init.phase", true);

- // Create a bunch of tables with columns key and value
- LinkedList<String> cols = new LinkedList<String>();
- cols.add("key");
- cols.add("value");
-
- LinkedList<String> part_cols = new LinkedList<String>();
- part_cols.add("ds");
- part_cols.add("hr");
- db.createTable("srcpart", cols, part_cols, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
-
- Path fpath;
- HashMap<String, String> part_spec = new HashMap<String, String>();
- for (String ds : new String[] {"2008-04-08", "2008-04-09"}) {
- for (String hr : new String[] {"11", "12"}) {
- part_spec.clear();
- part_spec.put("ds", ds);
- part_spec.put("hr", hr);
- // System.out.println("Loading partition with spec: " + part_spec);
- // db.createPartition(srcpart, part_spec);
- fpath = new Path(testFiles, "kv1.txt");
- // db.loadPartition(fpath, srcpart.getName(), part_spec, true);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='"
- + hr + "')");
- }
- }
- ArrayList<String> bucketCols = new ArrayList<String>();
- bucketCols.add("key");
- runCreateTableCmd("CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE");
- // db.createTable("srcbucket", cols, null, TextInputFormat.class,
- // IgnoreKeyTextOutputFormat.class, 2, bucketCols);
- for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) {
- fpath = new Path(testFiles, fname);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE srcbucket");
- }
-
- runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) "
- + "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE");
- // db.createTable("srcbucket", cols, null, TextInputFormat.class,
- // IgnoreKeyTextOutputFormat.class, 2, bucketCols);
- for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt",
- "srcbucket22.txt", "srcbucket23.txt"}) {
- fpath = new Path(testFiles, fname);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE srcbucket2");
- }
-
- for (String tname : new String[] {"src", "src1"}) {
- db.createTable(tname, cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- }
- db.createTable("src_sequencefile", cols, null,
- SequenceFileInputFormat.class, SequenceFileOutputFormat.class);
-
- Table srcThrift =
- new Table(SessionState.get().getCurrentDatabase(), "src_thrift");
- srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName());
- srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName());
- srcThrift.setSerializationLib(ThriftDeserializer.class.getName());
- srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_CLASS, Complex.class
- .getName());
- srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT,
- TBinaryProtocol.class.getName());
- db.createTable(srcThrift);
-
- LinkedList<String> json_cols = new LinkedList<String>();
- json_cols.add("json");
- db.createTable("src_json", json_cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
-
- // load the input data into the src table
- fpath = new Path(testFiles, "kv1.txt");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src");
-
- // load the input data into the src table
- fpath = new Path(testFiles, "kv3.txt");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src1");
-
- // load the input data into the src_sequencefile table
- fpath = new Path(testFiles, "kv1.seq");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE src_sequencefile");
-
- // load the input data into the src_thrift table
- fpath = new Path(testFiles, "complex.seq");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE src_thrift");
-
- // load the json data into the src_json table
- fpath = new Path(testFiles, "json.txt");
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE src_json");
-
- FileSystem localFs = FileSystem.getLocal(conf);
- // create and load data into orc table
- fpath = new Path(testFiles, AllVectorTypesRecord.TABLE_NAME);
-
- runCreateTableCmd(AllVectorTypesRecord.TABLE_CREATE_COMMAND);
- runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
- + "' INTO TABLE "+AllVectorTypesRecord.TABLE_NAME);
-
- runCmd("DROP FUNCTION IF EXISTS qtest_get_java_boolean ");
- runCmd("CREATE FUNCTION qtest_get_java_boolean "
- + " AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean'");
+ String initCommands = readEntireFileIntoString(new File(this.initScript));
+ LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ cliDriver.processLine("set test.data.dir=" + testFiles + ";");
+ cliDriver.processLine(initCommands);

      conf.setBoolean("hive.test.init.phase", false);
-
    }

    public void init() throws Exception {
@@ -786,33 +709,7 @@ public class QTestUtil {
    public void init(String tname) throws Exception {
      cleanUp();
      createSources();
-
- LinkedList<String> cols = new LinkedList<String>();
- cols.add("key");
- cols.add("value");
-
- LinkedList<String> part_cols = new LinkedList<String>();
- part_cols.add("ds");
- part_cols.add("hr");
-
- db.createTable("dest1", cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- db.createTable("dest2", cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
-
- db.createTable("dest3", cols, part_cols, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- Table dest3 = db.getTable("dest3");
-
- HashMap<String, String> part_spec = new HashMap<String, String>();
- part_spec.put("ds", "2008-04-08");
- part_spec.put("hr", "12");
- db.createPartition(dest3, part_spec);
-
- db.createTable("dest4", cols, null, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class);
- db.createTable("dest4_sequencefile", cols, null,
- SequenceFileInputFormat.class, SequenceFileOutputFormat.class);
+ cliDriver.processCmd("set hive.cli.print.header=true;");
    }

    public void cliInit(String tname) throws Exception {
@@ -866,23 +763,38 @@ public class QTestUtil {
      SessionState.start(ss);

      cliDriver = new CliDriver();
+
      if (tname.equals("init_file.q")) {
        ss.initFiles.add("../../data/scripts/test_init_file.sql");
      }
      cliDriver.processInitFiles(ss);
+
      return outf.getAbsolutePath();
    }

    private CliSessionState startSessionState()
- throws FileNotFoundException, UnsupportedEncodingException {
+ throws IOException {

      HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
          "org.apache.hadoop.hive.ql.security.DummyAuthenticator");

      CliSessionState ss = new CliSessionState(conf);
      assert ss != null;
+ ss.in = System.in;
+ ss.out = System.out;
+ ss.err = System.out;

+ SessionState oldSs = SessionState.get();
+ if (oldSs != null && clusterType == MiniClusterType.tez) {
+ oldSs.close();
+ }
+ if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
+ oldSs.out.close();
+ }
      SessionState.start(ss);
+
+ isSessionStateStarted = true;
+
      return ss;
    }

@@ -1571,7 +1483,7 @@ public class QTestUtil {
    {
      QTestUtil[] qt = new QTestUtil[qfiles.length];
      for (int i = 0; i < qfiles.length; i++) {
- qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20");
+ qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", "", "");
        qt[i].addFile(qfiles[i]);
        qt[i].clearTestSideEffects();
      }

Modified: hive/trunk/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestCliDriver.vm (original)
+++ hive/trunk/ql/src/test/templates/TestCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -37,9 +37,12 @@ public class $className extends TestCase

      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
      String hiveConfDir = "$hiveConfDir";
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
      try {
        String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hiveConfDir, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ hiveConfDir, hadoopVer, initScript, cleanupScript);

        // do a one time initialization
        qt.cleanUp();

Modified: hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm (original)
+++ hive/trunk/ql/src/test/templates/TestCompareCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -38,9 +38,12 @@ public class $className extends TestCase

      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
      String hiveConfDir = "$hiveConfDir";
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
      try {
        String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hiveConfDir, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ hiveConfDir, hadoopVer, initScript, cleanupScript);

        // do a one time initialization
        qt.cleanUp();

Modified: hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm (original)
+++ hive/trunk/ql/src/test/templates/TestNegativeCliDriver.vm Sat Aug 9 01:34:24 2014
@@ -35,10 +35,13 @@ public class $className extends TestCase

    static {
      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";

      try {
        String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
+ initScript, cleanupScript);
        // do a one time initialization
        qt.cleanUp();
        qt.createSources();

Modified: hive/trunk/ql/src/test/templates/TestParse.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestParse.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestParse.vm (original)
+++ hive/trunk/ql/src/test/templates/TestParse.vm Sat Aug 9 01:34:24 2014
@@ -35,10 +35,13 @@ public class $className extends TestCase

    static {
      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";

      try {
        String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
+ initScript, cleanupScript);
        qt.init(null);
      } catch (Exception e) {
        System.err.println("Exception: " + e.getMessage());

Modified: hive/trunk/ql/src/test/templates/TestParseNegative.vm
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/templates/TestParseNegative.vm?rev=1616913&r1=1616912&r2=1616913&view=diff
==============================================================================
--- hive/trunk/ql/src/test/templates/TestParseNegative.vm (original)
+++ hive/trunk/ql/src/test/templates/TestParseNegative.vm Sat Aug 9 01:34:24 2014
@@ -36,10 +36,13 @@ public class $className extends TestCase
    static {

      MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";

      try {
        String hadoopVer = "$hadoopVersion";
- qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer);
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
+ initScript, cleanupScript);
      } catch (Exception e) {
        System.err.println("Exception: " + e.getMessage());
        e.printStackTrace();

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedAug 9, '14 at 1:34a
activeAug 9, '14 at 1:34a
posts1
users1
websitehive.apache.org

1 user in discussion

Szehon: 1 post

People

Translate

site design / logo © 2021 Grokbase