FAQ
Repository: hive
Updated Branches:
   refs/heads/master 274847e2a -> 50b6d0c6b


HIVE-11903 : Add lock metrics to HS2 (Yongzhi Chen via Szehon)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/50b6d0c6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/50b6d0c6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/50b6d0c6

Branch: refs/heads/master
Commit: 50b6d0c6bbf3ba838d3851ccc7cad0575c39732c
Parents: 274847e
Author: Szehon Ho <szehon@cloudera.com>
Authored: Wed Sep 30 12:21:30 2015 -0700
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Wed Sep 30 12:21:30 2015 -0700

----------------------------------------------------------------------
  .../common/metrics/common/MetricsConstant.java | 5 ++
  .../hadoop/hive/ql/lockmgr/DbLockManager.java | 21 ++++++++
  .../zookeeper/ZooKeeperHiveLockManager.java | 41 ++++++++++++++++
  .../zookeeper/TestZookeeperLockManager.java | 50 ++++++++++++++++++++
  4 files changed, 117 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
index 13c3cf9..88a3c29 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
@@ -33,4 +33,9 @@ public class MetricsConstant {
    public static String JDO_ROLLBACK_TRANSACTIONS = "rollbacked_jdo_transactions";
    public static String JDO_COMMIT_TRANSACTIONS = "committed_jdo_transactions";
    public static String JDO_OPEN_TRANSACTIONS = "opened_jdo_transactions";
+
+ public static String METASTORE_HIVE_LOCKS = "metastore_hive_locks";
+ public static String ZOOKEEPER_HIVE_SHAREDLOCKS = "zookeeper_hive_sharedlocks";
+ public static String ZOOKEEPER_HIVE_EXCLUSIVELOCKS = "zookeeper_hive_exclusivelocks";
+ public static String ZOOKEEPER_HIVE_SEMISHAREDLOCKS = "zookeeper_hive_semisharedlocks";
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
index 82e227f..bb9da9d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.hive.ql.lockmgr;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
  import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
  import org.apache.hadoop.hive.metastore.IMetaStoreClient;
  import org.apache.hadoop.hive.metastore.api.*;
@@ -99,6 +102,16 @@ public class DbLockManager implements HiveLockManager{
          throw new LockException(ErrorMsg.LOCK_CANNOT_BE_ACQUIRED.getMsg());
        }
        acquiredLocks.add(hl);
+
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ try {
+ metrics.incrementCounter(MetricsConstant.METASTORE_HIVE_LOCKS);
+ } catch (Exception e) {
+ LOG.warn("Error Reporting hive client metastore lock operation to Metrics system", e);
+ }
+ }
+
        return res.getState();
      } catch (NoSuchTxnException e) {
        LOG.error("Metastore could not find txnid " + lock.getTxnid());
@@ -133,6 +146,14 @@ public class DbLockManager implements HiveLockManager{
        LOG.debug("Unlocking " + hiveLock);
        client.unlock(lockId);
        boolean removed = locks.remove(hiveLock);
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ try {
+ metrics.decrementCounter(MetricsConstant.METASTORE_HIVE_LOCKS);
+ } catch (Exception e) {
+ LOG.warn("Error Reporting hive client metastore unlock operation to Metrics system", e);
+ }
+ }
        LOG.debug("Removed a lock " + removed);
      } catch (NoSuchLockException e) {
        LOG.error("Metastore could find no record of lock " + JavaUtils.lockIdToString(lockId));

http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
index fb954d8..7c7a8d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
@@ -19,8 +19,12 @@
  package org.apache.hadoop.hive.ql.lockmgr.zookeeper;

  import com.google.common.annotations.VisibleForTesting;
+
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.ql.ErrorMsg;
  import org.apache.hadoop.hive.ql.lockmgr.*;
@@ -402,7 +406,25 @@ public class ZooKeeperHiveLockManager implements HiveLockManager {
          return null;
        }
      }
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ try {
+ switch(mode) {
+ case EXCLUSIVE:
+ metrics.incrementCounter(MetricsConstant.ZOOKEEPER_HIVE_EXCLUSIVELOCKS);
+ break;
+ case SEMI_SHARED:
+ metrics.incrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SEMISHAREDLOCKS);
+ break;
+ default:
+ metrics.incrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SHAREDLOCKS);
+ break;
+ }

+ } catch (Exception e) {
+ LOG.warn("Error Reporting hive client zookeeper lock operation to Metrics system", e);
+ }
+ }
      return new ZooKeeperHiveLock(res, key, mode);
    }

@@ -438,6 +460,7 @@ public class ZooKeeperHiveLockManager implements HiveLockManager {
    @VisibleForTesting
    static void unlockPrimitive(HiveLock hiveLock, String parent, CuratorFramework curatorFramework) throws LockException {
      ZooKeeperHiveLock zLock = (ZooKeeperHiveLock)hiveLock;
+ HiveLockMode lMode = hiveLock.getHiveLockMode();
      HiveLockObject obj = zLock.getHiveLockObject();
      String name = getLastObjectName(parent, obj);
      try {
@@ -448,6 +471,24 @@ public class ZooKeeperHiveLockManager implements HiveLockManager {
        if (children == null || children.isEmpty()) {
          curatorFramework.delete().forPath(name);
        }
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ try {
+ switch(lMode) {
+ case EXCLUSIVE:
+ metrics.decrementCounter(MetricsConstant.ZOOKEEPER_HIVE_EXCLUSIVELOCKS);
+ break;
+ case SEMI_SHARED:
+ metrics.decrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SEMISHAREDLOCKS);
+ break;
+ default:
+ metrics.decrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SHAREDLOCKS);
+ break;
+ }
+ } catch (Exception e) {
+ LOG.warn("Error Reporting hive client zookeeper unlock operation to Metrics system", e);
+ }
+ }
      } catch (KeeperException.NoNodeException nne) {
        //can happen in retrying deleting the zLock after exceptions like InterruptedException
        //or in a race condition where parent has already been deleted by other process when it

http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
index 4a1ef2e..7fcaa22 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
@@ -18,7 +18,14 @@

  package org.apache.hadoop.hive.ql.lockmgr.zookeeper;

+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockManagerCtx;
  import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
  import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject;
  import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
@@ -33,6 +40,9 @@ import org.junit.Before;
  import org.junit.After;
  import org.junit.Test;

+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
  public class TestZookeeperLockManager {

    private HiveConf conf;
@@ -110,5 +120,45 @@ public class TestZookeeperLockManager {
      conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, "9999");
      Assert.assertEquals("node1:5666,node2:9999,node3:9999", ZooKeeperHiveHelper.getQuorumServers(conf));
    }
+
+ @Test
+ public void testMetrics() throws Exception{
+ conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM, "localhost");
+ conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, String.valueOf(server.getPort()));
+ File workDir = new File(System.getProperty("test.tmp.dir"));
+ File jsonReportFile = new File(workDir, "json_reportingzk1");
+ jsonReportFile.delete();
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+ MetricsFactory.init(conf);
+
+ HiveLockManagerCtx ctx = new HiveLockManagerCtx(conf);
+ ZooKeeperHiveLockManager zMgr= new ZooKeeperHiveLockManager();
+ zMgr.setContext(ctx);
+ ZooKeeperHiveLock curLock = zMgr.lock(hiveLock, HiveLockMode.SHARED, false);
+ Thread.sleep(2000);
+ byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode rootNode = objectMapper.readTree(jsonData);
+ JsonNode countersNode = rootNode.path("counters");
+ JsonNode zkLockNode = countersNode.path("zookeeper_hive_sharedlocks");
+ JsonNode zkLockCountNode = zkLockNode.path("count");
+ Assert.assertTrue(zkLockCountNode.asInt() == 1);
+
+ zMgr.unlock(curLock);
+ Thread.sleep(2000);
+ jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
+ objectMapper = new ObjectMapper();
+ rootNode = objectMapper.readTree(jsonData);
+ countersNode = rootNode.path("counters");
+ zkLockNode = countersNode.path("zookeeper_hive_sharedlocks");
+ zkLockCountNode = zkLockNode.path("count");
+ Assert.assertTrue(zkLockCountNode.asInt() == 0);
+ zMgr.close();
+ }
+
  }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedSep 30, '15 at 7:22p
activeSep 30, '15 at 7:22p
posts1
users1
websitehive.apache.org

1 user in discussion

Szehon: 1 post

People

Translate

site design / logo © 2021 Grokbase