FAQ
Repository: hive
Updated Branches:
   refs/heads/master da947fab4 -> ce6293b3d


HIVE-12271 : Add metrics around HS2 query execution and job submission for Hive (Szehon, reviewed by Jimmy Xiang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ce6293b3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ce6293b3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ce6293b3

Branch: refs/heads/master
Commit: ce6293b3d8383cf73a6d960cafdd5e15483526b2
Parents: da947fa
Author: Szehon Ho <szehon@cloudera.com>
Authored: Tue Nov 17 14:48:59 2015 -0800
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Tue Nov 17 14:50:02 2015 -0800

----------------------------------------------------------------------
  common/pom.xml | 17 +++
  .../hive/common/metrics/LegacyMetrics.java | 27 +++--
  .../hive/common/metrics/common/Metrics.java | 28 ++++-
  .../common/metrics/common/MetricsConstant.java | 6 +
  .../common/metrics/common/MetricsScope.java | 33 ++++++
  .../metrics/metrics2/CodahaleMetrics.java | 41 +++++--
  .../apache/hadoop/hive/ql/log/PerfLogger.java | 27 +++++
  .../hive/common/metrics/MetricsTestUtils.java | 61 ++++++++++
  .../hive/common/metrics/TestLegacyMetrics.java | 46 ++++----
  .../metrics/metrics2/TestCodahaleMetrics.java | 8 +-
  itests/hive-unit/pom.xml | 7 ++
  .../hive/jdbc/miniHS2/TestHs2Metrics.java | 116 +++++++++++++++++++
  .../hadoop/hive/metastore/HiveMetaStore.java | 4 +-
  .../java/org/apache/hadoop/hive/ql/Driver.java | 1 -
  service/pom.xml | 7 ++
  .../hive/service/cli/operation/Operation.java | 43 +++++++
  .../service/cli/session/SessionManager.java | 25 +++-
  .../cli/session/TestSessionManagerMetrics.java | 100 ++++++++++++++++
  18 files changed, 542 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 2292fdf..cd14581 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -239,6 +239,23 @@
            </execution>
          </executions>
        </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <configuration>
+ <!--exclude configuration xml that might be picked up-->
+ <excludes>
+ <exclude>*.xml</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
      </plugins>
    </build>
  </project>

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
index 52d99e4..9be9b50 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
@@ -18,6 +18,7 @@
  package org.apache.hadoop.hive.common.metrics;

  import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsScope;
  import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
  import org.apache.hadoop.hive.conf.HiveConf;

@@ -56,7 +57,7 @@ public class LegacyMetrics implements Metrics {
     * (i) a "number of calls" counter ( &lt;name&gt;.n ), and
     * (ii) a "number of msecs spent between scope open and close" counter. ( &lt;name&gt;.t)
     */
- public static class MetricsScope {
+ public static class LegacyMetricsScope implements MetricsScope {

      final LegacyMetrics metrics;

@@ -73,7 +74,7 @@ public class LegacyMetrics implements Metrics {
       * @param name - name of the variable
       * @throws IOException
       */
- private MetricsScope(String name, LegacyMetrics metrics) throws IOException {
+ private LegacyMetricsScope(String name, LegacyMetrics metrics) throws IOException {
        this.metrics = metrics;
        this.name = name;
        this.numCounter = name + ".n";
@@ -150,11 +151,11 @@ public class LegacyMetrics implements Metrics {
      }
    }

- private static final ThreadLocal<HashMap<String, MetricsScope>> threadLocalScopes
- = new ThreadLocal<HashMap<String,MetricsScope>>() {
+ private static final ThreadLocal<HashMap<String, LegacyMetricsScope>> threadLocalScopes
+ = new ThreadLocal<HashMap<String, LegacyMetricsScope>>() {
      @Override
- protected HashMap<String,MetricsScope> initialValue() {
- return new HashMap<String,MetricsScope>();
+ protected HashMap<String, LegacyMetricsScope> initialValue() {
+ return new HashMap<String, LegacyMetricsScope>();
      }
    };

@@ -212,15 +213,15 @@ public class LegacyMetrics implements Metrics {
      return metrics.get(name);
    }

- public void startScope(String name) throws IOException{
+ public void startStoredScope(String name) throws IOException{
      if (threadLocalScopes.get().containsKey(name)) {
        threadLocalScopes.get().get(name).open();
      } else {
- threadLocalScopes.get().put(name, new MetricsScope(name, this));
+ threadLocalScopes.get().put(name, new LegacyMetricsScope(name, this));
      }
    }

- public MetricsScope getScope(String name) throws IOException {
+ public MetricsScope getStoredScope(String name) throws IOException {
      if (threadLocalScopes.get().containsKey(name)) {
        return threadLocalScopes.get().get(name);
      } else {
@@ -228,13 +229,19 @@ public class LegacyMetrics implements Metrics {
      }
    }

- public void endScope(String name) throws IOException{
+ public void endStoredScope(String name) throws IOException{
      if (threadLocalScopes.get().containsKey(name)) {
        threadLocalScopes.get().get(name).close();
      }
    }

+ public MetricsScope createScope(String name) throws IOException {
+ return new LegacyMetricsScope(name, this);
+ }

+ public void endScope(MetricsScope scope) throws IOException {
+ ((LegacyMetricsScope) scope).close();
+ }

    /**
     * Resets the static context state to initial.

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
index 49b2b32..4297233 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
@@ -28,20 +28,40 @@ import java.io.IOException;
   */
  public interface Metrics {

- //Must declare CTOR taking in HiveConf.
-
    /**
     * Deinitializes the Metrics system.
     */
    public void close() throws Exception;

    /**
+ *
+ * @param name starts a scope of a given name. Scopes is stored as thread-local variable.
+ * @throws IOException
+ */
+ public void startStoredScope(String name) throws IOException;
+
+ /**
+ * Closes the stored scope of a given name.
+ * Note that this must be called on the same thread as where the scope was started.
+ * @param name
+ * @throws IOException
+ */
+ public void endStoredScope(String name) throws IOException;
+
+ /**
+ * Create scope with given name and returns it.
     * @param name
+ * @return
     * @throws IOException
     */
- public void startScope(String name) throws IOException;
+ public MetricsScope createScope(String name) throws IOException;

- public void endScope(String name) throws IOException;
+ /**
+ * Close the given scope.
+ * @param scope
+ * @throws IOException
+ */
+ public void endScope(MetricsScope scope) throws IOException;

    //Counter-related methods


http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
index a5aa995..f18aa6e 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
@@ -38,4 +38,10 @@ public class MetricsConstant {
    public static final String ZOOKEEPER_HIVE_SHAREDLOCKS = "zookeeper_hive_sharedlocks";
    public static final String ZOOKEEPER_HIVE_EXCLUSIVELOCKS = "zookeeper_hive_exclusivelocks";
    public static final String ZOOKEEPER_HIVE_SEMISHAREDLOCKS = "zookeeper_hive_semisharedlocks";
+
+ public static final String EXEC_ASYNC_QUEUE_SIZE = "exec_async_queue_size";
+ public static final String EXEC_ASYNC_POOL_SIZE = "exec_async_pool_size";
+
+ public static final String OPERATION_PREFIX = "hs2_operation_";
+ public static final String COMPLETED_OPERATION_PREFIX = "hs2_completed_operation_";
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsScope.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsScope.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsScope.java
new file mode 100644
index 0000000..3d6a23e
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsScope.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.common;
+
+/**
+ * Metrics Scope to represent duration of an event.
+ *
+ * Implementation can capture information like the average duration of open scopes,
+ * number of open scopes, number of completed scopes.
+ *
+ * Scopes are created via the Metrics framework (see Metrics#createScope or Metrics$createStoredScope)
+ *
+ * Scope may be stored by the Metrics framework via 'storedScope' concept for further reference.
+ *
+ * In either case, it is the caller's responsibility to end the scope via the Metrics framework (see Metrics#endScope)
+ */
+public interface MetricsScope {
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
index 3db26af..cba1c5a 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
@@ -44,6 +44,7 @@ import com.google.common.collect.Lists;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.common.metrics.common.MetricsScope;
  import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.slf4j.Logger;
@@ -72,6 +73,7 @@ import java.util.concurrent.locks.ReentrantLock;
   */
  public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.common.Metrics {
    public static final String API_PREFIX = "api_";
+ public static final String ACTIVE_CALLS = "active_calls_";
    public static final Logger LOGGER = LoggerFactory.getLogger(CodahaleMetrics.class);

    public final MetricRegistry metricRegistry = new MetricRegistry();
@@ -86,15 +88,15 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
    private HiveConf conf;
    private final Set<Closeable> reporters = new HashSet<Closeable>();

- private final ThreadLocal<HashMap<String, MetricsScope>> threadLocalScopes
- = new ThreadLocal<HashMap<String,MetricsScope>>() {
+ private final ThreadLocal<HashMap<String, CodahaleMetricsScope>> threadLocalScopes
+ = new ThreadLocal<HashMap<String, CodahaleMetricsScope>>() {
      @Override
- protected HashMap<String,MetricsScope> initialValue() {
- return new HashMap<String,MetricsScope>();
+ protected HashMap<String, CodahaleMetricsScope> initialValue() {
+ return new HashMap<String, CodahaleMetricsScope>();
      }
    };

- public static class MetricsScope {
+ public static class CodahaleMetricsScope implements MetricsScope {

      final String name;
      final Timer timer;
@@ -108,7 +110,7 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
       * @param name - name of the variable
       * @throws IOException
       */
- private MetricsScope(String name, CodahaleMetrics metrics) throws IOException {
+ private CodahaleMetricsScope(String name, CodahaleMetrics metrics) throws IOException {
        this.name = name;
        this.metrics = metrics;
        this.timer = metrics.getTimer(name);
@@ -124,6 +126,7 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
        if (!isOpen) {
          isOpen = true;
          this.timerContext = timer.time();
+ metrics.incrementCounter(ACTIVE_CALLS + name);
        } else {
          throw new IOException("Scope named " + name + " is not closed, cannot be opened.");
        }
@@ -136,7 +139,7 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
      public void close() throws IOException {
        if (isOpen) {
          timerContext.close();
-
+ metrics.decrementCounter(ACTIVE_CALLS + name);
        } else {
          throw new IOException("Scope named " + name + " is not open, cannot be closed.");
        }
@@ -210,23 +213,41 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
    }

    @Override
- public void startScope(String name) throws IOException {
+ public void startStoredScope(String name) throws IOException {
      name = API_PREFIX + name;
      if (threadLocalScopes.get().containsKey(name)) {
        threadLocalScopes.get().get(name).open();
      } else {
- threadLocalScopes.get().put(name, new MetricsScope(name, this));
+ threadLocalScopes.get().put(name, new CodahaleMetricsScope(name, this));
      }
    }

    @Override
- public void endScope(String name) throws IOException {
+ public void endStoredScope(String name) throws IOException {
      name = API_PREFIX + name;
      if (threadLocalScopes.get().containsKey(name)) {
        threadLocalScopes.get().get(name).close();
+ threadLocalScopes.get().remove(name);
+ }
+ }
+
+ public MetricsScope getStoredScope(String name) throws IOException {
+ if (threadLocalScopes.get().containsKey(name)) {
+ return threadLocalScopes.get().get(name);
+ } else {
+ throw new IOException("No metrics scope named " + name);
      }
    }

+ public MetricsScope createScope(String name) throws IOException {
+ name = API_PREFIX + name;
+ return new CodahaleMetricsScope(name, this);
+ }
+
+ public void endScope(MetricsScope scope) throws IOException {
+ ((CodahaleMetricsScope) scope).close();
+ }
+
    @Override
    public Long incrementCounter(String name) throws IOException {
      return incrementCounter(name, 1L);

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
index 548d7db..1ef636c 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -18,11 +18,14 @@

  package org.apache.hadoop.hive.ql.log;

+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.util.ReflectionUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;

+import java.io.IOException;
  import java.util.HashMap;
  import java.util.Map;

@@ -116,6 +119,7 @@ public class PerfLogger {
      long startTime = System.currentTimeMillis();
      LOG.info("<PERFLOG method=" + method + " from=" + callerName + ">");
      startTimes.put(method, new Long(startTime));
+ beginMetrics(method);
    }
    /**
     * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
@@ -156,6 +160,8 @@ public class PerfLogger {
      sb.append(">");
      LOG.info(sb.toString());

+ endMetrics(method);
+
      return duration;
    }

@@ -193,4 +199,25 @@ public class PerfLogger {
      return duration;
    }

+ private void beginMetrics(String method) {
+ Metrics metrics = MetricsFactory.getInstance();
+ try {
+ if (metrics != null) {
+ metrics.startStoredScope(method);
+ }
+ } catch (IOException e) {
+ LOG.warn("Error recording metrics", e);
+ }
+ }
+
+ private void endMetrics(String method) {
+ Metrics metrics = MetricsFactory.getInstance();
+ try {
+ if (metrics != null) {
+ metrics.endStoredScope(method);
+ }
+ } catch (IOException e) {
+ LOG.warn("Error recording metrics", e);
+ }
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
new file mode 100644
index 0000000..fd420f7
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Assert;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Utilities for codahale metrics verification.
+ */
+public class MetricsTestUtils {
+
+ public static final MetricsCategory COUNTER = new MetricsCategory("counters", "count");
+ public static final MetricsCategory TIMER = new MetricsCategory("timers", "count");
+ public static final MetricsCategory GAUGE = new MetricsCategory("gauges", "value");
+
+ static class MetricsCategory {
+ String category;
+ String metricsHandle;
+ MetricsCategory(String category, String metricsHandle) {
+ this.category = category;
+ this.metricsHandle = metricsHandle;
+ }
+ }
+
+ public static void verifyMetricFile(File jsonReportFile, MetricsCategory category, String metricsName,
+ Object value) throws Exception {
+ JsonNode jsonNode = getJsonNode(jsonReportFile, category, metricsName);
+ Assert.assertEquals(jsonNode.asText(), value.toString());
+ }
+
+ private static JsonNode getJsonNode(File jsonReportFile, MetricsCategory category, String metricsName) throws Exception {
+ byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode rootNode = objectMapper.readTree(jsonData);
+ JsonNode categoryNode = rootNode.path(category.category);
+ JsonNode metricsNode = categoryNode.path(metricsName);
+ return metricsNode.path(category.metricsHandle);
+ }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
index c3e8282..a3fb04f 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
@@ -32,7 +32,7 @@ import javax.management.MBeanServer;
  import javax.management.ObjectName;

  import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.LegacyMetrics.MetricsScope;
+import org.apache.hadoop.hive.common.metrics.common.MetricsScope;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.junit.After;
  import org.junit.Before;
@@ -124,8 +124,8 @@ public class TestLegacyMetrics {

    @Test
    public void testScopeSingleThread() throws Exception {
- metrics.startScope(scopeName);
- final MetricsScope fooScope = metrics.getScope(scopeName);
+ metrics.startStoredScope(scopeName);
+ final LegacyMetrics.LegacyMetricsScope fooScope = (LegacyMetrics.LegacyMetricsScope) metrics.getStoredScope(scopeName);
      // the time and number counters become available only after the 1st
      // scope close:
      expectIOE(new Callable<Long>() {
@@ -151,15 +151,15 @@ public class TestLegacyMetrics {
        }
      });

- assertSame(fooScope, metrics.getScope(scopeName));
+ assertSame(fooScope, metrics.getStoredScope(scopeName));
      Thread.sleep(periodMs+ 1);
      // 1st close:
      // closing of open scope should be ok:
- metrics.endScope(scopeName);
+ metrics.endStoredScope(scopeName);
      expectIOE(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
- metrics.endScope(scopeName); // closing of closed scope not allowed
+ metrics.endStoredScope(scopeName); // closing of closed scope not allowed
          return null;
        }
      });
@@ -168,15 +168,15 @@ public class TestLegacyMetrics {
      final long t1 = fooScope.getTimeCounter().longValue();
      assertTrue(t1 > periodMs);

- assertSame(fooScope, metrics.getScope(scopeName));
+ assertSame(fooScope, metrics.getStoredScope(scopeName));

     // opening allowed after closing:
- metrics.startScope(scopeName);
+ metrics.startStoredScope(scopeName);
      // opening of already open scope not allowed:
      expectIOE(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
- metrics.startScope(scopeName);
+ metrics.startStoredScope(scopeName);
          return null;
        }
      });
@@ -184,7 +184,7 @@ public class TestLegacyMetrics {
      assertEquals(Long.valueOf(1), fooScope.getNumCounter());
      assertEquals(t1, fooScope.getTimeCounter().longValue());

- assertSame(fooScope, metrics.getScope(scopeName));
+ assertSame(fooScope, metrics.getStoredScope(scopeName));
      Thread.sleep(periodMs + 1);
      // Reopening (close + open) allowed in opened state:
      fooScope.reopen();
@@ -204,8 +204,8 @@ public class TestLegacyMetrics {

    @Test
    public void testScopeConcurrency() throws Exception {
- metrics.startScope(scopeName);
- MetricsScope fooScope = metrics.getScope(scopeName);
+ metrics.startStoredScope(scopeName);
+ LegacyMetrics.LegacyMetricsScope fooScope = (LegacyMetrics.LegacyMetricsScope) metrics.getStoredScope(scopeName);
      final int threads = 10;
      ExecutorService executorService = Executors.newFixedThreadPool(threads);
      for (int i=0; i<threads; i++) {
@@ -221,17 +221,17 @@ public class TestLegacyMetrics {
      executorService.shutdown();
      assertTrue(executorService.awaitTermination(periodMs * 3 * threads, TimeUnit.MILLISECONDS));

- fooScope = metrics.getScope(scopeName);
+ fooScope = (LegacyMetrics.LegacyMetricsScope) metrics.getStoredScope(scopeName);
      assertEquals(Long.valueOf(3 * threads), fooScope.getNumCounter());
      assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs * threads);
      Double avgT = (Double) metrics.get("foo.avg_t");
      assertTrue(avgT.doubleValue() > periodMs);
- metrics.endScope(scopeName);
+ metrics.endStoredScope(scopeName);
    }

    void testScopeImpl(int n) throws Exception {
- metrics.startScope(scopeName);
- final MetricsScope fooScope = metrics.getScope(scopeName);
+ metrics.startStoredScope(scopeName);
+ final LegacyMetrics.LegacyMetricsScope fooScope = (LegacyMetrics.LegacyMetricsScope) metrics.getStoredScope(scopeName);
        // cannot open scope that is already open:
      expectIOE(new Callable<Void>() {
        @Override
@@ -241,10 +241,10 @@ public class TestLegacyMetrics {
        }
      });

- assertSame(fooScope, metrics.getScope(scopeName));
+ assertSame(fooScope, metrics.getStoredScope(scopeName));
      Thread.sleep(periodMs+ 1);
      // 1st close:
- metrics.endScope(scopeName); // closing of open scope should be ok.
+ metrics.endStoredScope(scopeName); // closing of open scope should be ok.

      assertTrue(fooScope.getNumCounter().longValue() >= 1);
      final long t1 = fooScope.getTimeCounter().longValue();
@@ -253,15 +253,15 @@ public class TestLegacyMetrics {
      expectIOE(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
- metrics.endScope(scopeName); // closing of closed scope not allowed
+ metrics.endStoredScope(scopeName); // closing of closed scope not allowed
          return null;
        }
      });

- assertSame(fooScope, metrics.getScope(scopeName));
+ assertSame(fooScope, metrics.getStoredScope(scopeName));

     // opening allowed after closing:
- metrics.startScope(scopeName);
+ metrics.startStoredScope(scopeName);

      assertTrue(fooScope.getNumCounter().longValue() >= 1);
      assertTrue(fooScope.getTimeCounter().longValue() >= t1);
@@ -270,12 +270,12 @@ public class TestLegacyMetrics {
      expectIOE(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
- metrics.startScope(scopeName);
+ metrics.startStoredScope(scopeName);
          return null;
        }
      });

- assertSame(fooScope, metrics.getScope(scopeName));
+ assertSame(fooScope, metrics.getStoredScope(scopeName));
      Thread.sleep(periodMs + 1);
      // Reopening (close + open) allowed in opened state:
      fooScope.reopen();

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
index a3aa549..27825b1 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
@@ -77,8 +77,8 @@ public class TestCodahaleMetrics {
    public void testScope() throws Exception {
      int runs = 5;
      for (int i = 0; i < runs; i++) {
- MetricsFactory.getInstance().startScope("method1");
- MetricsFactory.getInstance().endScope("method1");
+ MetricsFactory.getInstance().startStoredScope("method1");
+ MetricsFactory.getInstance().endStoredScope("method1");
      }

      Timer timer = metricRegistry.getTimers().get("api_method1");
@@ -106,8 +106,8 @@ public class TestCodahaleMetrics {
        executorService.submit(new Callable<Void>() {
          @Override
          public Void call() throws Exception {
- MetricsFactory.getInstance().startScope("method2");
- MetricsFactory.getInstance().endScope("method2");
+ MetricsFactory.getInstance().startStoredScope("method2");
+ MetricsFactory.getInstance().endStoredScope("method2");
            return null;
          }
        });

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/itests/hive-unit/pom.xml
----------------------------------------------------------------------
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index c202001..1809cd8 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -146,6 +146,13 @@
        <version>${project.version}</version>
        <classifier>tests</classifier>
      </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
      <!-- test inter-project -->
      <dependency>
        <groupId>junit</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHs2Metrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHs2Metrics.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHs2Metrics.java
new file mode 100644
index 0000000..873e126
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/miniHS2/TestHs2Metrics.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.jdbc.miniHS2;
+
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook;
+import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hive.service.cli.CLIServiceClient;
+import org.apache.hive.service.cli.SessionHandle;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests HiveServer2 metrics.
+ */
+public class TestHs2Metrics {
+
+ private static MiniHS2 miniHS2;
+ private static Map<String, String> confOverlay;
+ private static File jsonReportFile;
+
+ //Check metrics during semantic analysis.
+ public static class MetricCheckingHook implements HiveSemanticAnalyzerHook {
+ @Override
+ public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context,
+ ASTNode ast) throws SemanticException {
+ try {
+ //Pre-analyze hook is fired in the middle of these calls
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "active_calls_api_semanticAnalyze", 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "active_calls_api_compile", 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "active_calls_api_hs2_operation_RUNNING", 1);
+ } catch (Exception e) {
+ throw new SemanticException("metrics verification failed", e);
+ }
+ return ast;
+ }
+
+ @Override
+ public void postAnalyze(HiveSemanticAnalyzerHookContext context,
+ List<Task<? extends Serializable>> rootTasks) throws SemanticException {
+ }
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ miniHS2 = new MiniHS2(new HiveConf());
+ confOverlay = new HashMap<String, String>();
+ confOverlay.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ confOverlay.put(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, MetricCheckingHook.class.getName());
+ miniHS2.start(confOverlay);
+
+ //for Metrics. MiniHS2 init code-path doesn't go through HiveServer2.startHiveServer2().
+ File workDir = new File(System.getProperty("test.tmp.dir"));
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+ HiveConf conf = new HiveConf();
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+ MetricsFactory.init(conf);
+ }
+
+ @Test
+ public void testMetrics() throws Exception {
+ String tableName = "testMetrics";
+ CLIServiceClient serviceClient = miniHS2.getServiceClient();
+ SessionHandle sessHandle = serviceClient.openSession("foo", "bar");
+
+ //Block on semantic analysis to check 'active_calls'
+ serviceClient.executeStatement(sessHandle, "CREATE TABLE " + tableName + " (id INT)", confOverlay);
+ Thread.sleep(2000);
+
+ //check that all calls were recorded.
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.TIMER, "api_hs2_operation_INITIALIZED", 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.TIMER, "api_hs2_operation_PENDING", 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.TIMER, "api_hs2_operation_RUNNING", 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "hs2_completed_operation_FINISHED", 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.TIMER, "api_Driver.run", 1);
+
+ //but there should be no more active calls.
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "active_calls_api_semanticAnalyze", 0);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "active_calls_api_compile", 0);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, "active_calls_api_hs2_operation_RUNNING", 0);
+
+ }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 3c40d6e..a835f6a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -676,7 +676,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
            function + extraLogInfo);
        if (MetricsFactory.getInstance() != null) {
          try {
- MetricsFactory.getInstance().startScope(function);
+ MetricsFactory.getInstance().startStoredScope(function);
          } catch (IOException e) {
            LOG.debug("Exception when starting metrics scope"
              + e.getClass().getName() + " " + e.getMessage(), e);
@@ -720,7 +720,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
      private void endFunction(String function, MetaStoreEndFunctionContext context) {
        if (MetricsFactory.getInstance() != null) {
          try {
- MetricsFactory.getInstance().endScope(function);
+ MetricsFactory.getInstance().endStoredScope(function);
          } catch (IOException e) {
            LOG.debug("Exception when closing metrics scope" + e);
          }

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index ceaafe1..fc5a951 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1552,7 +1552,6 @@ public class Driver implements CommandProcessor {
          // Launch upto maxthreads tasks
          Task<? extends Serializable> task;
          while ((task = driverCxt.getRunnable(maxthreads)) != null) {
- perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TASK + task.getName() + "." + task.getId());
            TaskRunner runner = launchTask(task, queryId, noName, jobname, jobs, driverCxt);
            if (!runner.isRunning()) {
              break;

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/service/pom.xml
----------------------------------------------------------------------
diff --git a/service/pom.xml b/service/pom.xml
index 22234d9..afa52cf 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -121,6 +121,13 @@
        <scope>test</scope>
        <classifier>tests</classifier>
      </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
      <!-- test inter-project -->
      <dependency>
        <groupId>junit</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index a851936..d13415e 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -19,13 +19,17 @@ package org.apache.hive.service.cli.operation;

  import java.io.File;
  import java.io.FileNotFoundException;
+import java.io.IOException;
  import java.util.EnumSet;
+import java.util.Set;
  import java.util.concurrent.Future;
  import java.util.concurrent.TimeUnit;

+import com.google.common.collect.Sets;
  import org.apache.hadoop.hive.common.metrics.common.Metrics;
  import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
  import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.common.metrics.common.MetricsScope;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.apache.hadoop.hive.conf.HiveConf;
@@ -51,6 +55,7 @@ public abstract class Operation {

    protected final HiveSession parentSession;
    private OperationState state = OperationState.INITIALIZED;
+ private MetricsScope currentStateScope;
    private final OperationHandle opHandle;
    private HiveConf configuration;
    public static final FetchOrientation DEFAULT_FETCH_ORIENTATION = FetchOrientation.FETCH_NEXT;
@@ -76,6 +81,7 @@ public abstract class Operation {
      lastAccessTime = System.currentTimeMillis();
      operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(),
          HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
+ setMetrics(state);
    }

    public Future<?> getBackgroundHandle() {
@@ -134,6 +140,7 @@ public abstract class Operation {
    protected final OperationState setState(OperationState newState) throws HiveSQLException {
      state.validateTransition(newState);
      this.state = newState;
+ setMetrics(state);
      this.lastAccessTime = System.currentTimeMillis();
      return this.state;
    }
@@ -353,4 +360,40 @@ public abstract class Operation {
      }
      return ex;
    }
+
+ //list of operation states to measure duration of.
+ protected static Set<OperationState> scopeStates = Sets.immutableEnumSet(
+ OperationState.INITIALIZED,
+ OperationState.PENDING,
+ OperationState.RUNNING
+ );
+
+ //list of terminal operation states. We measure only completed counts for operations in these states.
+ protected static Set<OperationState> terminalStates = Sets.immutableEnumSet(
+ OperationState.CLOSED,
+ OperationState.CANCELED,
+ OperationState.FINISHED,
+ OperationState.ERROR,
+ OperationState.UNKNOWN
+ );
+
+ protected void setMetrics(OperationState state) {
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ try {
+ if (currentStateScope != null) {
+ metrics.endScope(currentStateScope);
+ currentStateScope = null;
+ }
+ if (scopeStates.contains(state)) {
+ currentStateScope = metrics.createScope(MetricsConstant.OPERATION_PREFIX + state.toString());
+ }
+ if (terminalStates.contains(state)) {
+ metrics.incrementCounter(MetricsConstant.COMPLETED_OPERATION_PREFIX + state.toString());
+ }
+ } catch (IOException e) {
+ LOG.warn("Error metrics", e);
+ }
+ }
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
index a9b4334..d11cf3d 100644
--- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
+++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -25,13 +25,19 @@ import java.util.ArrayList;
  import java.util.Date;
  import java.util.List;
  import java.util.Map;
+import java.util.concurrent.BlockingQueue;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.Future;
  import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
  import java.util.concurrent.ThreadPoolExecutor;
  import java.util.concurrent.TimeUnit;

  import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.apache.hadoop.hive.conf.HiveConf;
@@ -108,8 +114,9 @@ public class SessionManager extends CompositeService {
      // Threads terminate when they are idle for more than the keepAliveTime
      // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize
      String threadPoolName = "HiveServer2-Background-Pool";
+ final BlockingQueue queue = new LinkedBlockingQueue<Runnable>(poolQueueSize);
      backgroundOperationPool = new ThreadPoolExecutor(poolSize, poolSize,
- keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(poolQueueSize),
+ keepAliveTime, TimeUnit.SECONDS, queue,
          new ThreadFactoryWithGarbageCleanup(threadPoolName));
      backgroundOperationPool.allowCoreThreadTimeOut(true);

@@ -119,6 +126,22 @@ public class SessionManager extends CompositeService {
          hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
      checkOperation = HiveConf.getBoolVar(hiveConf,
          ConfVars.HIVE_SERVER2_IDLE_SESSION_CHECK_OPERATION);
+
+ Metrics m = MetricsFactory.getInstance();
+ if (m != null) {
+ m.addGauge(MetricsConstant.EXEC_ASYNC_QUEUE_SIZE, new MetricsVariable() {
+ @Override
+ public Object getValue() {
+ return queue.size();
+ }
+ });
+ m.addGauge(MetricsConstant.EXEC_ASYNC_POOL_SIZE, new MetricsVariable() {
+ @Override
+ public Object getValue() {
+ return backgroundOperationPool.getPoolSize();
+ }
+ });
+ }
    }

    private void initOperationLogRootDir() {

http://git-wip-us.apache.org/repos/asf/hive/blob/ce6293b3/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java b/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java
new file mode 100644
index 0000000..aaeecbe
--- /dev/null
+++ b/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.session;
+
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.service.server.HiveServer2;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+
+/**
+ * Test metrics from SessionManager.
+ */
+public class TestSessionManagerMetrics {
+
+ private static SessionManager sm;
+ private static File jsonReportFile;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ HiveConf conf = new HiveConf();
+ conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS, 2);
+ conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE, 10);
+ conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, "1000000s");
+
+ File workDir = new File(System.getProperty("test.tmp.dir"));
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+ MetricsFactory.init(conf);
+
+ HiveServer2 hs2 = new HiveServer2();
+ sm = new SessionManager(hs2);
+ sm.init(conf);
+ }
+
+ final Object barrier = new Object();
+
+ class BarrierRunnable implements Runnable {
+ @Override
+ public void run() {
+ synchronized (barrier) {
+ try {
+ barrier.wait();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Tests metrics regarding async thread pool.
+ */
+ @Test
+ public void testThreadPoolMetrics() throws Exception {
+
+ sm.submitBackgroundOperation(new BarrierRunnable());
+ sm.submitBackgroundOperation(new BarrierRunnable());
+ sm.submitBackgroundOperation(new BarrierRunnable());
+ sm.submitBackgroundOperation(new BarrierRunnable());
+
+ Thread.sleep(2000);
+
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.EXEC_ASYNC_POOL_SIZE, 2);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.EXEC_ASYNC_QUEUE_SIZE, 2);
+
+ synchronized (barrier) {
+ barrier.notifyAll();
+ }
+ Thread.sleep(2000);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.EXEC_ASYNC_POOL_SIZE, 2);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.EXEC_ASYNC_QUEUE_SIZE, 0);
+ }
+}

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 17, '15 at 10:52p
activeNov 17, '15 at 10:52p
posts1
users1
websitehive.apache.org

1 user in discussion

Szehon: 1 post

People

Translate

site design / logo © 2021 Grokbase