FAQ
Repository: hive
Updated Branches:
   refs/heads/master 1cb4ce871 -> 37e6e1bf5


HIVE-13149: Remove some unnecessary HMS connections from HS2 (Reviewed by Szehon Ho, Chaoyu Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37e6e1bf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37e6e1bf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37e6e1bf

Branch: refs/heads/master
Commit: 37e6e1bf56d7d2fd557730380b147c745fc051ce
Parents: 1cb4ce8
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Fri Mar 18 10:27:49 2016 -0400
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Apr 11 14:58:59 2016 -0400

----------------------------------------------------------------------
  .../hadoop/hive/metastore/TestMetastoreVersion.java | 7 ++++---
  .../metastore/hbase/TestHBaseMetastoreMetrics.java | 4 +---
  .../org/apache/hadoop/hive/hbase/HBaseQTestUtil.java | 10 +++++++++-
  .../org/apache/hadoop/hive/hbase/HBaseTestSetup.java | 3 ---
  .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 14 ++++++++++----
  .../hadoop/hive/metastore/HiveMetaStoreClient.java | 10 ++++++----
  .../apache/hadoop/hive/ql/session/SessionState.java | 8 --------
  7 files changed, 30 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 53f0d0e..5ceb3d2 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore;

  import java.io.File;
  import java.lang.reflect.Field;
-import java.util.Random;

  import junit.framework.TestCase;

@@ -32,6 +31,7 @@ import org.apache.hive.common.util.HiveStringUtils;
  import org.apache.hadoop.hive.metastore.api.MetaException;
  import org.apache.hadoop.hive.metastore.ObjectStore;
  import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.metadata.Hive;
  import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
  import org.apache.hadoop.hive.ql.session.SessionState;

@@ -96,8 +96,9 @@ public class TestMetastoreVersion extends TestCase {
      // session creation should fail since the schema didn't get created
      try {
        SessionState.start(new CliSessionState(hiveConf));
- fail("Expected exception");
- } catch (RuntimeException re) {
+ Hive.get(hiveConf).getMSC();
+ fail("An exception is expected since schema is not created.");
+ } catch (Exception re) {
        LOG.info("Exception in testVersionRestriction: " + re, re);
        String msg = HiveStringUtils.stringifyException(re);
        assertTrue("Expected 'Version information not found in metastore' in: " + msg, msg

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
index 3ed88f2..aefafe0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
@@ -41,8 +41,6 @@ import java.io.IOException;
   */
  public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {

- private CodahaleMetrics metrics;
-
    @BeforeClass
    public static void startup() throws Exception {
      HBaseIntegrationTests.startMiniCluster();
@@ -66,7 +64,6 @@ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
      conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
      SessionState.start(new CliSessionState(conf));
      driver = new Driver(conf);
- metrics = (CodahaleMetrics) MetricsFactory.getInstance();
    }

    @Test
@@ -107,6 +104,7 @@ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
      driver.run("use default");
      driver.run("drop database tempdb cascade");

+ CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
      String json = metrics.dumpJson();
      MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_DATABASES, 2);
      MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_TABLES, 7);

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 3ff5742..70c0b13 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -39,12 +39,14 @@ public class HBaseQTestUtil extends QTestUtil {
    /** A handle to this harness's cluster */
    private final HConnection conn;

+ private HBaseTestSetup setup;
+
    public HBaseQTestUtil(
      String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup,
      String initScript, String cleanupScript)
      throws Exception {
-
      super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false);
+ this.setup = setup;
      setup.preTest(conf);
      this.conn = setup.getConnection();
      super.init();
@@ -69,6 +71,12 @@ public class HBaseQTestUtil extends QTestUtil {
    }

    @Override
+ protected void initConfFromSetup() throws Exception {
+ super.initConfFromSetup();
+ setup.preTest(conf);
+ }
+
+ @Override
    public void createSources(String tname) throws Exception {
      super.createSources(tname);


http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
index e6383dc..cee7158 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
@@ -22,9 +22,6 @@ import java.io.IOException;
  import java.net.ServerSocket;
  import java.util.Arrays;

-import junit.extensions.TestSetup;
-import junit.framework.Test;
-
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hbase.HBaseConfiguration;
  import org.apache.hadoop.hbase.HColumnDescriptor;

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 8473436..2f109ab 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -510,6 +510,7 @@ public class QTestUtil {
        dfs.shutdown();
        dfs = null;
      }
+ Hive.closeCurrent();
    }

    public String readEntireFileIntoString(File queryFile) throws IOException {
@@ -731,8 +732,9 @@ public class QTestUtil {
        return;
      }

- db.getConf().set("hive.metastore.filter.hook",
+ conf.set("hive.metastore.filter.hook",
          "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl");
+ db = Hive.get(conf);
      // Delete any tables other than the source tables
      // and any databases other than the default database.
      for (String dbName : db.getAllDatabases()) {
@@ -800,16 +802,20 @@ public class QTestUtil {
        return;
      }

- clearTablesCreatedDuringTests();
- clearKeysCreatedInTests();
-
      // allocate and initialize a new conf since a test can
      // modify conf by using 'set' commands
      conf = new HiveConf(Driver.class);
      initConf();
+ initConfFromSetup();
+
      // renew the metastore since the cluster type is unencrypted
      db = Hive.get(conf); // propagate new conf to meta store

+ clearTablesCreatedDuringTests();
+ clearKeysCreatedInTests();
+ }
+
+ protected void initConfFromSetup() throws Exception {
      setup.preTest(conf);
    }


http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index cdd12ab..64a26ac 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -178,7 +178,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
    private boolean isConnected = false;
    private URI metastoreUris[];
    private final HiveMetaHookLoader hookLoader;
- protected final HiveConf conf;
+ protected final HiveConf conf; // Keep a copy of HiveConf so if Session conf changes, we may need to get a new HMS client.
    protected boolean fastpath = false;
    private String tokenStrForm;
    private final boolean localMetaStore;
@@ -205,8 +205,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
      this.hookLoader = hookLoader;
      if (conf == null) {
        conf = new HiveConf(HiveMetaStoreClient.class);
+ this.conf = conf;
+ } else {
+ this.conf = new HiveConf(conf);
      }
- this.conf = conf;
      filterHook = loadFilterHooks();
      fileMetadataBatchSize = HiveConf.getIntVar(
          conf, HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX);
@@ -221,10 +223,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
        // instantiate the metastore server handler directly instead of connecting
        // through the network
        if (conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) {
- client = new HiveMetaStore.HMSHandler("hive client", conf, true);
+ client = new HiveMetaStore.HMSHandler("hive client", this.conf, true);
          fastpath = true;
        } else {
- client = HiveMetaStore.newRetryingHMSHandler("hive client", conf, true);
+ client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true);
        }
        isConnected = true;
        snapshotActiveConf();

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 23b8a96..f7a0f31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -546,10 +546,6 @@ public class SessionState {
      // Get the following out of the way when you start the session these take a
      // while and should be done when we start up.
      try {
- // Hive object instance should be created with a copy of the conf object. If the conf is
- // shared with SessionState, other parts of the code might update the config, but
- // Hive.get(HiveConf) would not recognize the case when it needs refreshing
- Hive.get(new HiveConf(startSs.conf)).getMSC();
        UserGroupInformation sessionUGI = Utils.getUGI();
        FileSystem.get(startSs.conf);

@@ -575,10 +571,6 @@ public class SessionState {
        }
      } catch (RuntimeException e) {
        throw e;
- } catch (Hive.SchemaException e) {
- RuntimeException ex = new RuntimeException(e.getMessage());
- ex.setStackTrace(new StackTraceElement[0]);
- throw ex;
      } catch (Exception e) {
        // Catch-all due to some exec time dependencies on session state
        // that would cause ClassNoFoundException otherwise

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 11, '16 at 7:00p
activeApr 11, '16 at 7:00p
posts1
users1
websitehive.apache.org

1 user in discussion

Aihuaxu: 1 post

People

Translate

site design / logo © 2021 Grokbase