Grokbase Groups Hive commits May 2015
FAQ
Repository: hive
Updated Branches:
   refs/heads/master 652febcda -> e2a12c9a6


HIVE-9508: MetaStore client socket connection should have a lifetime (Thiruvel Thirumoolan reviewed by Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2a12c9a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2a12c9a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2a12c9a

Branch: refs/heads/master
Commit: e2a12c9a630d037f3e3aaf42acc873eac86bc9f3
Parents: 652febc
Author: Vaibhav Gumashta <vgumashta@apache.org>
Authored: Tue May 5 10:44:16 2015 -0700
Committer: Vaibhav Gumashta <vgumashta@apache.org>
Committed: Tue May 5 10:44:16 2015 -0700

----------------------------------------------------------------------
  .../org/apache/hadoop/hive/conf/HiveConf.java | 6 ++++
  .../hive/metastore/TestHiveMetaStore.java | 30 ++++++++++++++++++++
  .../hive/metastore/RetryingMetaStoreClient.java | 27 +++++++++++++++---
  3 files changed, 59 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5d4dbea..69fda45 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -119,6 +119,7 @@ public class HiveConf extends Configuration {
        HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES,
        HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY,
        HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
+ HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME,
        HiveConf.ConfVars.METASTOREPWD,
        HiveConf.ConfVars.METASTORECONNECTURLHOOK,
        HiveConf.ConfVars.METASTORECONNECTURLKEY,
@@ -398,6 +399,11 @@ public class HiveConf extends Configuration {
      METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", "600s",
          new TimeValidator(TimeUnit.SECONDS),
          "MetaStore Client socket timeout in seconds"),
+ METASTORE_CLIENT_SOCKET_LIFETIME("hive.metastore.client.socket.lifetime", "0s",
+ new TimeValidator(TimeUnit.SECONDS),
+ "MetaStore Client socket lifetime in seconds. After this time is exceeded, client\n" +
+ "reconnects on the next MetaStore operation. A value of 0s means the connection\n" +
+ "has an infinite lifetime."),
      METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine",
          "password to use against metastore database"),
      METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "",

http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 130fd67..dffeb34 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2867,4 +2867,34 @@ public abstract class TestHiveMetaStore extends TestCase {
          ownerName, ownerType, createTime, functionType, resources);
      client.createFunction(func);
    }
+
+ public void testRetriableClientWithConnLifetime() throws Exception {
+
+ HiveConf conf = new HiveConf(hiveConf);
+ conf.setLong(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME.name(), 60);
+ long timeout = 65 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect
+
+ // Test a normal retriable client
+ IMetaStoreClient client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName());
+ client.getAllDatabases();
+ client.close();
+
+ // Connect after the lifetime, there should not be any failures
+ client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName());
+ Thread.sleep(timeout);
+ client.getAllDatabases();
+ client.close();
+ }
+
+ private HiveMetaHookLoader getHookLoader() {
+ HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+ @Override
+ public HiveMetaHook getHook(
+ org.apache.hadoop.hive.metastore.api.Table tbl)
+ throws MetaException {
+ return null;
+ }
+ };
+ return hookLoader;
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
index 77da6f7..1b6487a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
@@ -54,9 +54,9 @@ public class RetryingMetaStoreClient implements InvocationHandler {
    private final int retryLimit;
    private final long retryDelaySeconds;
    private final Map<String, Long> metaCallTimeMap;
-
-
-
+ private final long connectionLifeTimeInMillis;
+ private long lastConnectionTime;
+ private boolean localMetaStore;

    protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
        Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException {
@@ -64,6 +64,11 @@ public class RetryingMetaStoreClient implements InvocationHandler {
      this.retryDelaySeconds = hiveConf.getTimeVar(
          HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
      this.metaCallTimeMap = metaCallTimeMap;
+ this.connectionLifeTimeInMillis =
+ hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000;
+ this.lastConnectionTime = System.currentTimeMillis();
+ String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS);
+ localMetaStore = (msUri == null) || msUri.trim().isEmpty();

      reloginExpiringKeytabUser();
      this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] {
@@ -104,8 +109,9 @@ public class RetryingMetaStoreClient implements InvocationHandler {
      while (true) {
        try {
          reloginExpiringKeytabUser();
- if(retriesMade > 0){
+ if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) {
            base.reconnect();
+ lastConnectionTime = System.currentTimeMillis();
          }
          if (metaCallTimeMap == null) {
            ret = method.invoke(base, args);
@@ -171,6 +177,19 @@ public class RetryingMetaStoreClient implements InvocationHandler {
      return methodSb.toString();
    }

+ private boolean hasConnectionLifeTimeReached(Method method) {
+ if (connectionLifeTimeInMillis <= 0 || localMetaStore ||
+ method.getName().equalsIgnoreCase("close")) {
+ return false;
+ }
+ boolean shouldReconnect =
+ (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect);
+ }
+ return shouldReconnect;
+ }
+
    /**
     * Relogin if login user is logged in using keytab
     * Relogin is actually done by ugi code only if sufficient time has passed

Search Discussions

Discussion Posts

Previous

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 5, '15 at 5:43p
activeMay 5, '15 at 5:44p
posts2
users1
websitehive.apache.org

1 user in discussion

Vgumashta: 2 posts

People

Translate

site design / logo © 2021 Grokbase