FAQ
Author: vgumashta
Date: Wed Nov 5 03:27:59 2014
New Revision: 1636795

URL: http://svn.apache.org/r1636795
Log:
HIVE-8723: Set reasonable connection timeout for CuratorFramework ZooKeeper clients in Hive (Vaibhav Gumashta reviewed by Thejas Nair)

Modified:
     hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
     hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/Utils.java
     hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
     hive/branches/branch-0.14/service/src/java/org/apache/hive/service/server/HiveServer2.java
     hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java

Modified: hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java?rev=1636795&r1=1636794&r2=1636795&view=diff
==============================================================================
--- hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java (original)
+++ hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java Wed Nov 5 03:27:59 2014
@@ -43,9 +43,6 @@ public class TestZooKeeperTokenStore ext
    private CuratorFramework zkClient = null;
    private int zkPort = -1;
    private ZooKeeperTokenStore ts;
- // connect timeout large enough for slower test environments
- private final int connectTimeoutMillis = 30000;
- private final int sessionTimeoutMillis = 3000;

    @Override
    protected void setUp() throws Exception {
@@ -55,10 +52,9 @@ public class TestZooKeeperTokenStore ext
      }
      this.zkCluster = new MiniZooKeeperCluster();
      this.zkPort = this.zkCluster.startup(zkDataDir);
-
- this.zkClient = CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort)
- .sessionTimeoutMs(sessionTimeoutMillis).connectionTimeoutMs(connectTimeoutMillis)
- .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
+ this.zkClient =
+ CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort)
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
      this.zkClient.start();
    }

@@ -74,15 +70,9 @@ public class TestZooKeeperTokenStore ext

    private Configuration createConf(String zkPath) {
      Configuration conf = new Configuration();
- conf.set(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR,
- "localhost:" + this.zkPort);
- conf.set(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE,
- zkPath);
- conf.setLong(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS,
- connectTimeoutMillis);
+ conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, "localhost:"
+ + this.zkPort);
+ conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, zkPath);
      return conf;
    }


Modified: hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/Utils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/Utils.java?rev=1636795&r1=1636794&r2=1636795&view=diff
==============================================================================
--- hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/Utils.java (original)
+++ hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/Utils.java Wed Nov 5 03:27:59 2014
@@ -100,9 +100,6 @@ public class Utils {
      static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";

      // Non-configurable params:
- // ZOOKEEPER_SESSION_TIMEOUT is not exposed as client configurable
- static final int ZOOKEEPER_SESSION_TIMEOUT = 600 * 1000;
- static final int ZOOKEEPER_CONNECTION_TIMEOUT = -1;
      // Currently supports JKS keystore format
      static final String SSL_TRUST_STORE_TYPE = "JKS";


Modified: hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java?rev=1636795&r1=1636794&r2=1636795&view=diff
==============================================================================
--- hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java (original)
+++ hive/branches/branch-0.14/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java Wed Nov 5 03:27:59 2014
@@ -63,8 +63,6 @@ public class ZooKeeperHiveClientHelper {
      String serverNode;
      CuratorFramework zooKeeperClient =
          CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble)
- .sessionTimeoutMs(JdbcConnectionParams.ZOOKEEPER_SESSION_TIMEOUT)
- .connectionTimeoutMs(JdbcConnectionParams.ZOOKEEPER_CONNECTION_TIMEOUT)
              .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
      zooKeeperClient.start();
      try {

Modified: hive/branches/branch-0.14/service/src/java/org/apache/hive/service/server/HiveServer2.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/service/src/java/org/apache/hive/service/server/HiveServer2.java?rev=1636795&r1=1636794&r2=1636795&view=diff
==============================================================================
--- hive/branches/branch-0.14/service/src/java/org/apache/hive/service/server/HiveServer2.java (original)
+++ hive/branches/branch-0.14/service/src/java/org/apache/hive/service/server/HiveServer2.java Wed Nov 5 03:27:59 2014
@@ -143,9 +143,6 @@ public class HiveServer2 extends Composi
     * @throws Exception
     */
    private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception {
- int zooKeeperSessionTimeout =
- hiveConf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT);
- int connectTimeoutMillis = -1;
      String zooKeeperEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
      String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE);
      String instanceURI = getServerInstanceURI(hiveConf);
@@ -155,7 +152,6 @@ public class HiveServer2 extends Composi
      // Use the zooKeeperAclProvider to create appropriate ACLs
      zooKeeperClient =
          CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble)
- .sessionTimeoutMs(zooKeeperSessionTimeout).connectionTimeoutMs(connectTimeoutMillis)
              .aclProvider(zooKeeperAclProvider).retryPolicy(new ExponentialBackoffRetry(1000, 3))
              .build();
      zooKeeperClient.start();
@@ -345,14 +341,10 @@ public class HiveServer2 extends Composi
     */
    static void deleteServerInstancesFromZooKeeper(String versionNumber) throws Exception {
      HiveConf hiveConf = new HiveConf();
- int zooKeeperSessionTimeout =
- hiveConf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT);
- int connectTimeoutMillis = -1;
      String zooKeeperEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
      String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE);
      CuratorFramework zooKeeperClient =
          CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble)
- .sessionTimeoutMs(zooKeeperSessionTimeout).connectionTimeoutMs(connectTimeoutMillis)
              .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
      zooKeeperClient.start();
      List<String> znodePaths =

Modified: hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java?rev=1636795&r1=1636794&r2=1636795&view=diff
==============================================================================
--- hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java (original)
+++ hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java Wed Nov 5 03:27:59 2014
@@ -62,8 +62,7 @@ public class ZooKeeperTokenStore impleme
    private String rootNode = "";
    private volatile CuratorFramework zkSession;
    private String zkConnectString;
- private final int zkSessionTimeout = 3000;
- private int connectTimeoutMillis = -1;
+ private int connectTimeoutMillis;
    private List<ACL> newNodeAcl = Arrays.asList(new ACL(Perms.ALL, Ids.AUTH_IDS));

    /**
@@ -101,10 +100,10 @@ public class ZooKeeperTokenStore impleme
      if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) {
        synchronized (this) {
          if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) {
- zkSession = CuratorFrameworkFactory.builder().connectString(zkConnectString)
- .sessionTimeoutMs(zkSessionTimeout).connectionTimeoutMs(connectTimeoutMillis)
- .aclProvider(aclDefaultProvider)
- .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
+ zkSession =
+ CuratorFrameworkFactory.builder().connectString(zkConnectString)
+ .connectionTimeoutMs(connectTimeoutMillis).aclProvider(aclDefaultProvider)
+ .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
            zkSession.start();
          }
        }
@@ -431,12 +430,14 @@ public class ZooKeeperTokenStore impleme
    @Override
    public void init(Object objectStore, ServerMode smode) {
      this.serverMode = smode;
- zkConnectString = conf.get(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null);
+ zkConnectString =
+ conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null);
      if (zkConnectString == null || zkConnectString.trim().isEmpty()) {
        // try alternate config param
- zkConnectString = conf.get(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE, null);
+ zkConnectString =
+ conf.get(
+ HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE,
+ null);
        if (zkConnectString == null || zkConnectString.trim().isEmpty()) {
          throw new IllegalArgumentException("Zookeeper connect string has to be specifed through "
              + "either " + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR
@@ -445,14 +446,17 @@ public class ZooKeeperTokenStore impleme
              + WHEN_ZK_DSTORE_MSG);
        }
      }
- connectTimeoutMillis = conf.getInt(
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS, -1);
+ connectTimeoutMillis =
+ conf.getInt(
+ HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS,
+ CuratorFrameworkFactory.builder().getConnectionTimeoutMs());
      String aclStr = conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL, null);
      if (StringUtils.isNotBlank(aclStr)) {
        this.newNodeAcl = parseACLs(aclStr);
      }
- rootNode = conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE,
- HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT) + serverMode;
+ rootNode =
+ conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE,
+ HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT) + serverMode;

      try {
        // Install the JAAS Configuration for the runtime

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 5, '14 at 3:29a
activeNov 5, '14 at 3:29a
posts1
users1
websitehive.apache.org

1 user in discussion

Vgumashta: 1 post

People

Translate

site design / logo © 2021 Grokbase