FAQ
Repository: hive
Updated Branches:
   refs/heads/master 1b99efec6 -> f8f50ab17


HIVE-12967: Change LlapServiceDriver to read a properties file instead of llap-daemon-site (Siddarth Seth, via Gopal V)

Signed-off-by: Gopal V <gopalv@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8f50ab1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8f50ab1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8f50ab1

Branch: refs/heads/master
Commit: f8f50ab17da3155489e96cdf2e9b849aa88b8700
Parents: 1b99efe
Author: Gopal V <gopalv@apache.org>
Authored: Tue Feb 16 16:10:34 2016 -0800
Committer: Gopal V <gopalv@apache.org>
Committed: Tue Feb 16 16:26:04 2016 -0800

----------------------------------------------------------------------
  bin/ext/llap.sh | 1 +
  .../org/apache/hadoop/hive/conf/HiveConf.java | 78 +++++++++-
  .../llap/configuration/LlapConfiguration.java | 41 -----
  .../registry/impl/LlapFixedRegistryImpl.java | 6 +-
  .../registry/impl/LlapYarnRegistryImpl.java | 5 +-
  .../hive/llap/cli/LlapOptionsProcessor.java | 96 ++++++++----
  .../hadoop/hive/llap/cli/LlapServiceDriver.java | 155 ++++++++++++-------
  .../configuration/LlapDaemonConfiguration.java | 39 +++++
  .../hive/llap/daemon/impl/AMReporter.java | 1 -
  .../hive/llap/daemon/impl/LlapDaemon.java | 4 +-
  .../hive/llap/daemon/impl/QueryTracker.java | 2 -
  .../llap/shufflehandler/ShuffleHandler.java | 21 ++-
  .../main/resources/llap-cli-log4j2.properties | 77 +++++++++
  .../hive/llap/daemon/MiniLlapCluster.java | 1 -
  .../impl/TestLlapDaemonProtocolServerImpl.java | 4 +-
  15 files changed, 376 insertions(+), 155 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/bin/ext/llap.sh
----------------------------------------------------------------------
diff --git a/bin/ext/llap.sh b/bin/ext/llap.sh
index fa87587..838bb3a 100644
--- a/bin/ext/llap.sh
+++ b/bin/ext/llap.sh
@@ -30,6 +30,7 @@ llap () {

    set -e;

+ export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=llap-cli-log4j2.properties "
    # hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf
    $HADOOP $CLASS $HIVE_OPTS -directory $TMPDIR "$@"


http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 3ab1dba..48e8491 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.conf;
  import com.google.common.base.Joiner;
  import org.apache.commons.lang.StringUtils;
  import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
  import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
  import org.apache.hadoop.hive.conf.Validator.PatternSet;
  import org.apache.hadoop.hive.conf.Validator.RangeValidator;
@@ -46,9 +47,11 @@ import java.io.InputStream;
  import java.io.PrintStream;
  import java.net.URL;
  import java.util.ArrayList;
+import java.util.Collections;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Iterator;
+import java.util.LinkedHashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
@@ -109,8 +112,17 @@ public class HiveConf extends Configuration {
      for (ConfVars confVar : ConfVars.values()) {
        vars.put(confVar.varname, confVar);
      }
+
+ Set<String> llapDaemonConfVarsSetLocal = new LinkedHashSet<>();
+ populateLlapDaemonVarsSet(llapDaemonConfVarsSetLocal);
+ llapDaemonVarsSet = Collections.unmodifiableSet(llapDaemonConfVarsSetLocal);
    }

+ @InterfaceAudience.Private
+ public static final String PREFIX_LLAP = "llap.";
+ @InterfaceAudience.Private
+ public static final String PREFIX_HIVE_LLAP = "hive.llap.";
+
    /**
     * Metastore related options that the db is initialized against. When a conf
     * var in this is list is changed, the metastore instance for the CLI will
@@ -237,6 +249,70 @@ public class HiveConf extends Configuration {
    };

    /**
+ * Variables used by LLAP daemons.
+ * TODO: Eventually auto-populate this based on prefixes. The conf variables
+ * will need to be renamed for this.
+ */
+ private static final Set<String> llapDaemonVarsSet;
+
+ private static void populateLlapDaemonVarsSet(Set<String> llapDaemonVarsSetLocal) {
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_ENABLED.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_MEMORY_MODE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_MAX_ALLOC.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_ARENA_COUNT.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ALLOCATOR_DIRECT.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_USE_LRFU.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_LRFU_LAMBDA.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_USE_FILEID_PATH.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ORC_ENABLE_TIME_COUNTERS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_THREADPOOL_SIZE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_PRINCIPAL.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_KEYTAB_FILE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_KERBEROS_PRINCIPAL.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_KERBEROS_KEYTAB_FILE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_ZKSM_ZK_CONNECTION_STRING.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_SECURITY_ACL.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_MANAGEMENT_ACL.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DELEGATION_TOKEN_LIFETIME.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_MANAGEMENT_RPC_PORT.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_WEB_AUTO_AUTH.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WORK_DIRS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_TIMEOUT_MS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_SLEEP_BETWEEN_RETRIES_MS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_RPC_PORT.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_NUM_FILE_CLEANER_THREADS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_SERVICE_REFRESH_INTERVAL.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WAIT_QUEUE_COMPARATOR_CLASS_NAME.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_PORT.varname);
+ llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_WEB_SSL.varname);
+ }
+
+ /**
+ * Get a set containing configuration parameter names used by LLAP Server isntances
+ * @return an unmodifiable set containing llap ConfVars
+ */
+ public static final Set<String> getLlapDaemonConfVars() {
+ return llapDaemonVarsSet;
+ }
+
+
+ /**
     * ConfVars.
     *
     * These are the default configuration properties for Hive. Each HiveConf
@@ -2492,7 +2568,7 @@ public class HiveConf extends Configuration {
      LLAP_DAEMON_YARN_SHUFFLE_PORT("hive.llap.daemon.yarn.shuffle.port", 15551,
        "YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"),
      LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1,
- "TODO doc. Unused?", "llap.daemon.yarn.container.mb"),
+ "llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"),
      LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED("hive.llap.daemon.shuffle.dir.watcher.enabled", false,
        "TODO doc", "llap.daemon.shuffle.dir-watcher.enabled"),
      LLAP_DAEMON_AM_LIVENESS_HEARTBEAT_INTERVAL_MS(

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
deleted file mode 100644
index abdbc09..0000000
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.configuration;
-
-import java.net.URL;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class LlapConfiguration extends Configuration {
- public static final String LLAP_PREFIX = "llap.";
- public static final String LLAP_DAEMON_PREFIX = "llap.daemon.";
-
- public LlapConfiguration(Configuration conf) {
- super(conf);
- addResource(LLAP_DAEMON_SITE);
- }
-
- public LlapConfiguration() {
- super(false);
- addResource(LLAP_DAEMON_SITE);
- }
-
- public LlapConfiguration(Configuration conf, URL llapDaemonConfLocation) {
- super(conf);
- addResource(llapDaemonConfLocation);
- }
-
- private static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml";
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
index 92044bb..c3c16c4 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
  import org.apache.hadoop.hive.llap.registry.ServiceInstance;
  import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
  import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
@@ -67,9 +66,8 @@ public class LlapFixedRegistryImpl implements ServiceRegistry {
      this.mngPort = HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT);

      for (Map.Entry<String, String> kv : conf) {
- if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)
- || kv.getKey().startsWith("hive.llap.")
- || kv.getKey().startsWith(LlapConfiguration.LLAP_PREFIX)) {
+ if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP)
+ || kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
          // TODO: read this somewhere useful, like the task scheduler
          srv.put(kv.getKey(), kv.getValue());
        }

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
index efe31cc..c83dd6e 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapYarnRegistryImpl.java
@@ -39,7 +39,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
  import org.apache.hadoop.hive.llap.registry.ServiceInstance;
  import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
  import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
@@ -161,8 +160,8 @@ public class LlapYarnRegistryImpl implements ServiceRegistry {
      srv.addExternalEndpoint(getServicesEndpoint());

      for (Map.Entry<String, String> kv : this.conf) {
- if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)
- || kv.getKey().startsWith("hive.llap.")) {
+ if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP)
+ || kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
          // TODO: read this somewhere useful, like the task scheduler
          srv.set(kv.getKey(), kv.getValue());
        }

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index 6d25384..1732df8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -18,8 +18,9 @@

  package org.apache.hadoop.hive.llap.cli;

-import java.util.HashMap;
-import java.util.Map;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
  import java.util.Properties;

  import javax.annotation.Nonnull;
@@ -36,6 +37,21 @@ import org.apache.hadoop.util.StringUtils;

  public class LlapOptionsProcessor {

+ public static final String OPTION_INSTANCES = "instances"; //forward as arg
+ public static final String OPTION_NAME = "name"; // forward as arg
+ public static final String OPTION_DIRECTORY = "directory"; // work-dir
+ public static final String OPTION_ARGS = "args"; // forward as arg
+ public static final String OPTION_LOGLEVEL = "loglevel"; // forward as arg
+ public static final String OPTION_CHAOS_MONKEY = "chaosmonkey"; // forward as arg
+ public static final String OPTION_EXECUTORS = "executors"; // llap-daemon-site
+ public static final String OPTION_CACHE = "cache"; // llap-daemon-site
+ public static final String OPTION_SIZE = "size"; // forward via config.json
+ public static final String OPTION_XMX = "xmx"; // forward as arg
+ public static final String OPTION_AUXJARS = "auxjars"; // used to localize jars
+ public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars
+ public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json
+ public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter
+
    public class LlapOptions {
      private final int instances;
      private final String directory;
@@ -47,10 +63,12 @@ public class LlapOptionsProcessor {
      private final String jars;
      private final boolean isHbase;
      private final Properties conf;
+ private final String javaPath;

      public LlapOptions(String name, int instances, String directory, int executors, long cache,
- long size, long xmx, String jars, boolean isHbase, @Nonnull Properties hiveconf)
- throws ParseException {
+ long size, long xmx, String jars, boolean isHbase,
+ @Nonnull Properties hiveconf, String javaPath)
+ throws ParseException {
        if (instances <= 0) {
          throw new ParseException("Invalid configuration: " + instances
              + " (should be greater than 0)");
@@ -65,6 +83,7 @@ public class LlapOptionsProcessor {
        this.jars = jars;
        this.isHbase = isHbase;
        this.conf = hiveconf;
+ this.javaPath = javaPath;
      }

      public String getName() {
@@ -106,57 +125,67 @@ public class LlapOptionsProcessor {
      public Properties getConfig() {
        return conf;
      }
+
+ public String getJavaPath() {
+ return javaPath;
+ }
    }

    protected static final Logger l4j = LoggerFactory.getLogger(LlapOptionsProcessor.class.getName());
    private final Options options = new Options();
- Map<String, String> hiveVariables = new HashMap<String, String>();
    private org.apache.commons.cli.CommandLine commandLine;

    @SuppressWarnings("static-access")
    public LlapOptionsProcessor() {

      // set the number of instances on which llap should run
- options.addOption(OptionBuilder.hasArg().withArgName("instances").withLongOpt("instances")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_INSTANCES).withLongOpt(OPTION_INSTANCES)
          .withDescription("Specify the number of instances to run this on").create('i'));

- options.addOption(OptionBuilder.hasArg().withArgName("name").withLongOpt("name")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_NAME).withLongOpt(OPTION_NAME)
          .withDescription("Cluster name for YARN registry").create('n'));

- options.addOption(OptionBuilder.hasArg().withArgName("directory").withLongOpt("directory")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_DIRECTORY).withLongOpt(OPTION_DIRECTORY)
          .withDescription("Temp directory for jars etc.").create('d'));

- options.addOption(OptionBuilder.hasArg().withArgName("args").withLongOpt("args")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_ARGS).withLongOpt(OPTION_ARGS)
          .withDescription("java arguments to the llap instance").create('a'));

- options.addOption(OptionBuilder.hasArg().withArgName("loglevel").withLongOpt("loglevel")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_LOGLEVEL).withLongOpt(OPTION_LOGLEVEL)
          .withDescription("log levels for the llap instance").create('l'));

- options.addOption(OptionBuilder.hasArg().withArgName("chaosmonkey").withLongOpt("chaosmonkey")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CHAOS_MONKEY).withLongOpt(OPTION_CHAOS_MONKEY)
          .withDescription("chaosmonkey interval").create('m'));

- options.addOption(OptionBuilder.hasArg().withArgName("executors").withLongOpt("executors")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_EXECUTORS).withLongOpt(OPTION_EXECUTORS)
          .withDescription("executor per instance").create('e'));

- options.addOption(OptionBuilder.hasArg().withArgName("cache").withLongOpt("cache")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CACHE).withLongOpt(OPTION_CACHE)
          .withDescription("cache size per instance").create('c'));

- options.addOption(OptionBuilder.hasArg().withArgName("size").withLongOpt("size")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SIZE).withLongOpt(OPTION_SIZE)
          .withDescription("container size per instance").create('s'));

- options.addOption(OptionBuilder.hasArg().withArgName("xmx").withLongOpt("xmx")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_XMX).withLongOpt(OPTION_XMX)
          .withDescription("working memory size").create('w'));

- options.addOption(OptionBuilder.hasArg().withArgName("auxjars").withLongOpt("auxjars")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXJARS).withLongOpt(OPTION_AUXJARS)
          .withDescription("additional jars to package (by default, JSON SerDe jar is packaged"
              + " if available)").create('j'));

- options.addOption(OptionBuilder.hasArg().withArgName("auxhbase").withLongOpt("auxhbase")
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_AUXHBASE).withLongOpt(OPTION_AUXHBASE)
          .withDescription("whether to package the HBase jars (true by default)").create('h'));

+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_JAVA_HOME).withLongOpt(OPTION_JAVA_HOME)
+ .withDescription(
+ "Path to the JRE/JDK. This should be installed at the same location on all cluster nodes ($JAVA_HOME, java.home by default)")
+ .create());
+
      // -hiveconf x=y
      options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value")
- .withLongOpt("hiveconf").withDescription("Use value for given property").create());
+ .withLongOpt(OPTION_HIVECONF)
+ .withDescription("Use value for given property. Overridden by explicit parameters")
+ .create());

      // [-H|--help]
      options.addOption(new Option("H", "help", false, "Print help information"));
@@ -166,7 +195,7 @@ public class LlapOptionsProcessor {
      return StringUtils.TraditionalBinaryPrefix.string2long(value);
    }

- public LlapOptions processOptions(String argv[]) throws ParseException {
+ public LlapOptions processOptions(String argv[]) throws ParseException, IOException {
      commandLine = new GnuParser().parse(options, argv);
      if (commandLine.hasOption('H') || false == commandLine.hasOption("instances")) {
        // needs at least --instances
@@ -174,31 +203,34 @@ public class LlapOptionsProcessor {
        return null;
      }

- int instances = Integer.parseInt(commandLine.getOptionValue("instances"));
- String directory = commandLine.getOptionValue("directory");
- String jars = commandLine.getOptionValue("auxjars");
+ int instances = Integer.parseInt(commandLine.getOptionValue(OPTION_INSTANCES));
+ String directory = commandLine.getOptionValue(OPTION_DIRECTORY);
+ String jars = commandLine.getOptionValue(OPTION_AUXJARS);

- String name = commandLine.getOptionValue("name", null);
+ String name = commandLine.getOptionValue(OPTION_NAME, null);

- final int executors = Integer.parseInt(commandLine.getOptionValue("executors", "-1"));
- final long cache = parseSuffixed(commandLine.getOptionValue("cache", "-1"));
- final long size = parseSuffixed(commandLine.getOptionValue("size", "-1"));
- final long xmx = parseSuffixed(commandLine.getOptionValue("xmx", "-1"));
- final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue("auxhbase", "true"));
+ final int executors = Integer.parseInt(commandLine.getOptionValue(OPTION_EXECUTORS, "-1"));
+ final long cache = parseSuffixed(commandLine.getOptionValue(OPTION_CACHE, "-1"));
+ final long size = parseSuffixed(commandLine.getOptionValue(OPTION_SIZE, "-1"));
+ final long xmx = parseSuffixed(commandLine.getOptionValue(OPTION_XMX, "-1"));
+ final boolean isHbase = Boolean.parseBoolean(commandLine.getOptionValue(OPTION_AUXHBASE, "true"));

      final Properties hiveconf;

- if (commandLine.hasOption("hiveconf")) {
- hiveconf = commandLine.getOptionProperties("hiveconf");
+ if (commandLine.hasOption(OPTION_HIVECONF)) {
+ hiveconf = commandLine.getOptionProperties(OPTION_HIVECONF);
      } else {
        hiveconf = new Properties();
      }

+ String javaHome = null;
+ if (commandLine.hasOption(OPTION_JAVA_HOME)) {
+ javaHome = commandLine.getOptionValue(OPTION_JAVA_HOME);
+ }
      // loglevel, chaosmonkey & args are parsed by the python processor

      return new LlapOptions(
- name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf);
-
+ name, instances, directory, executors, cache, size, xmx, jars, isHbase, hiveconf, javaHome);
    }

    private void printUsage() {

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index d01c8ce..32c791f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -21,12 +21,14 @@ package org.apache.hadoop.hive.llap.cli;
  import java.io.IOException;
  import java.io.OutputStreamWriter;
  import java.net.URL;
-import java.util.Map;
  import java.util.Map.Entry;
  import java.util.Collection;
+import java.util.Properties;

+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
  import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
  import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
+import org.apache.tez.dag.api.TezConfiguration;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.apache.hadoop.conf.Configuration;
@@ -52,14 +54,18 @@ import com.google.common.base.Preconditions;
  public class LlapServiceDriver {

    protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName());
+
    private static final String[] DEFAULT_AUX_CLASSES = new String[] {
    "org.apache.hive.hcatalog.data.JsonSerDe" };
    private static final String HBASE_SERDE_CLASS = "org.apache.hadoop.hive.hbase.HBaseSerDe";
- private static final String[] NEEDED_CONFIGS = {
- "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" };
+ private static final String[] NEEDED_CONFIGS = LlapDaemonConfiguration.DAEMON_CONFIGS;
    private static final String[] OPTIONAL_CONFIGS = { "ssl-server.xml" };


+ /**
+ * This is a working configuration for the instance to merge various variables.
+ * It is not written out for llap server usage
+ */
    private final Configuration conf;

    public LlapServiceDriver() {
@@ -82,40 +88,56 @@ public class LlapServiceDriver {
      System.exit(ret);
    }

- /**
- * Intersect llap-daemon-site.xml configuration properties against an existing Configuration
- * object, while resolving any ${} parameters that might be present.
- *
- * @param raw
- * @return configuration object which is a slice of configured
- */
- public static Configuration resolve(Configuration configured, String first, String... resources) {
- Configuration defaults = new Configuration(false);

- defaults.addResource(first);
+ private static Configuration resolve(Configuration configured, Properties direct,
+ Properties hiveconf) {
+ Configuration conf = new Configuration(false);

- for (String resource : resources) {
- defaults.addResource(resource);
- }
+ populateConf(configured, conf, hiveconf, "CLI hiveconf");
+ populateConf(configured, conf, direct, "CLI direct");
+
+ return conf;
+ }

- Configuration slice = new Configuration(false);
- // for everything in defaults, slice out those from the configured
- for (Map.Entry<String, String> kv : defaults) {
- slice.set(kv.getKey(), configured.get(kv.getKey()));
+ private static void populateConf(Configuration configured, Configuration target,
+ Properties properties, String source) {
+ for (Entry<Object, Object> entry : properties.entrySet()) {
+ String key = (String) entry.getKey();
+ String val = configured.get(key);
+ if (val != null) {
+ target.set(key, val, source);
+ }
      }
+ }

- return slice;
+ private static void populateConfWithLlapProperties(Configuration conf, Properties properties) {
+ for(Entry<Object, Object> props : properties.entrySet()) {
+ String key = (String) props.getKey();
+ if (HiveConf.getLlapDaemonConfVars().contains(key)) {
+ conf.set(key, (String) props.getValue());
+ } else {
+ if (key.startsWith(HiveConf.PREFIX_LLAP) || key.startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
+ LOG.warn("Adding key [{}] even though it is not in the set of known llap-server keys");
+ conf.set(key, (String) props.getValue());
+ } else {
+ LOG.warn("Ignoring unknown llap server parameter: [{}]", key);
+ }
+ }
+ }
    }

    private void run(String[] args) throws Exception {
      LlapOptionsProcessor optionsProcessor = new LlapOptionsProcessor();
      LlapOptions options = optionsProcessor.processOptions(args);

+ Properties propsDirectOptions = new Properties();
+
      if (options == null) {
        // help
        return;
      }

+ // Working directory.
      Path tmpDir = new Path(options.getDirectory());

      if (conf == null) {
@@ -135,14 +157,19 @@ public class LlapServiceDriver {
      for (String f : OPTIONAL_CONFIGS) {
        conf.addResource(f);
      }
+
      conf.reloadConfiguration();

+ populateConfWithLlapProperties(conf, options.getConfig());
+
+
      if (options.getName() != null) {
        // update service registry configs - caveat: this has nothing to do with the actual settings
        // as read by the AM
        // if needed, use --hiveconf llap.daemon.service.hosts=@llap0 to dynamically switch between
        // instances
        conf.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + options.getName());
+ propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + options.getName());
      }

      if (options.getSize() != -1) {
@@ -160,35 +187,40 @@ public class LlapServiceDriver {
        }
      }

+ // This parameter is read in package.py - and nowhere else. Does not need to be part of HiveConf - that's just confusing.
      final long minAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1);
+ long containerSize = -1;
      if (options.getSize() != -1) {
- final long containerSize = options.getSize() / (1024 * 1024);
+ containerSize = options.getSize() / (1024 * 1024);
        Preconditions.checkArgument(containerSize >= minAlloc,
            "Container size should be greater than minimum allocation(%s)", minAlloc + "m");
        conf.setLong(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSize);
+ propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, String.valueOf(containerSize));
      }

      if (options.getExecutors() != -1) {
        conf.setLong(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, options.getExecutors());
+ propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, String.valueOf(options.getExecutors()));
        // TODO: vcpu settings - possibly when DRFA works right
      }

      if (options.getCache() != -1) {
        conf.set(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
            Long.toString(options.getCache()));
+ propsDirectOptions.setProperty(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
+ Long.toString(options.getCache()));
      }

      if (options.getXmx() != -1) {
        // Needs more explanation here
        // Xmx is not the max heap value in JDK8
- // You need to subtract 50% of the survivor fraction from this, to get actual usable memory before it goes into GC
- conf.setLong(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, (long)(options.getXmx())
- / (1024 * 1024));
+ // You need to subtract 50% of the survivor fraction from this, to get actual usable memory before it goes into GC
+ long xmx = (long) (options.getXmx() / (1024 * 1024));
+ conf.setLong(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, xmx);
+ propsDirectOptions.setProperty(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, String.valueOf(xmx));
      }

- for (Entry<Object, Object> props : options.getConfig().entrySet()) {
- conf.set((String) props.getKey(), (String) props.getValue());
- }
+

      URL logger = conf.getResource("llap-daemon-log4j2.properties");

@@ -205,9 +237,10 @@ public class LlapServiceDriver {
        LOG.warn("Unable to find llap scripts:" + scripts);
      }

+
      Path libDir = new Path(tmpDir, "lib");

- String tezLibs = conf.get("tez.lib.uris");
+ String tezLibs = conf.get(TezConfiguration.TEZ_LIB_URIS);
      if (tezLibs == null) {
        LOG.warn("Missing tez.lib.uris in tez-site.xml");
      }
@@ -264,31 +297,43 @@ public class LlapServiceDriver {
        }
      }

+ String java_home;
+ if (options.getJavaPath() == null || options.getJavaPath().isEmpty()) {
+ java_home = System.getenv("JAVA_HOME");
+ String jre_home = System.getProperty("java.home");
+ if (java_home == null) {
+ java_home = jre_home;
+ } else if (!java_home.equals(jre_home)) {
+ LOG.warn("Java versions might not match : JAVA_HOME=[{}],process jre=[{}]",
+ java_home, jre_home);
+ }
+ } else {
+ java_home = options.getJavaPath();
+ }
+ if (java_home == null || java_home.isEmpty()) {
+ throw new RuntimeException(
+ "Could not determine JAVA_HOME from command line parameters, environment or system properties");
+ }
+ LOG.info("Using [{}] for JAVA_HOME", java_home);
+
      Path confPath = new Path(tmpDir, "conf");
      lfs.mkdirs(confPath);

+ // Copy over the mandatory configs for the package.
      for (String f : NEEDED_CONFIGS) {
- copyConfig(options, lfs, confPath, f);
+ copyConfig(lfs, confPath, f);
      }
      for (String f : OPTIONAL_CONFIGS) {
        try {
- copyConfig(options, lfs, confPath, f);
+ copyConfig(lfs, confPath, f);
        } catch (Throwable t) {
          LOG.info("Error getting an optional config " + f + "; ignoring: " + t.getMessage());
        }
      }
+ createLlapDaemonConfig(lfs, confPath, conf, propsDirectOptions, options.getConfig());

      lfs.copyFromLocalFile(new Path(logger.toString()), confPath);

- String java_home = System.getenv("JAVA_HOME");
- String jre_home = System.getProperty("java.home");
- if (java_home == null) {
- java_home = jre_home;
- } else if (!java_home.equals(jre_home)) {
- LOG.warn("Java versions might not match : JAVA_HOME=%s,process jre=%s",
- java_home, jre_home);
- }
-
      // extract configs for processing by the python fragments in Slider
      JSONObject configs = new JSONObject();

@@ -296,6 +341,7 @@ public class LlapServiceDriver {

      configs.put(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, HiveConf.getIntVar(conf,
          ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB));
+ configs.put(ConfVars.LLAP_DAEMON_YARN_CONTAINER_MB.varname, containerSize);

      configs.put(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
          HiveConf.getSizeVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE));
@@ -362,23 +408,22 @@ public class LlapServiceDriver {
      }
    }

- private void copyConfig(
- LlapOptions options, FileSystem lfs, Path confPath, String f) throws IOException {
- if (f.equals("llap-daemon-site.xml")) {
- FSDataOutputStream confStream = lfs.create(new Path(confPath, f));
+ private void createLlapDaemonConfig(FileSystem lfs, Path confPath, Configuration configured,
+ Properties direct, Properties hiveconf) throws IOException {
+ FSDataOutputStream confStream =
+ lfs.create(new Path(confPath, LlapDaemonConfiguration.LLAP_DAEMON_SITE));

- Configuration copy = resolve(conf, "llap-daemon-site.xml");
+ Configuration llapDaemonConf = resolve(configured, direct, hiveconf);

- for (Entry<Object, Object> props : options.getConfig().entrySet()) {
- // overrides
- copy.set((String) props.getKey(), (String) props.getValue());
- }
+ llapDaemonConf.writeXml(confStream);
+ confStream.close();
+ }

- copy.writeXml(confStream);
- confStream.close();
- } else {
- // they will be file:// URLs
- lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
- }
+ private void copyConfig(FileSystem lfs, Path confPath, String f) throws IOException {
+ HiveConf.getBoolVar(new Configuration(false), ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS);
+ // they will be file:// URLs
+ lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
    }
+
+
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java b/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
new file mode 100644
index 0000000..72340f5
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.configuration;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
+/**
+ * Configuration for LLAP daemon processes only. This should not be used by any clients.
+ */
+public class LlapDaemonConfiguration extends Configuration {
+
+ @InterfaceAudience.Private
+ public static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml";
+
+ @InterfaceAudience.Private
+ public static final String[] DAEMON_CONFIGS = { /* in specific order */"core-site.xml",
+ "hdfs-site.xml", "yarn-site.xml", "tez-site.xml", "hive-site.xml" };
+
+ public LlapDaemonConfiguration() {
+ super(false);
+ for (String conf : DAEMON_CONFIGS) {
+ addResource(conf);
+ }
+ addResource(LLAP_DAEMON_SITE);
+ }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index d1ec715..04c28cb 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
  import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler;
  import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
  import org.apache.hadoop.io.Text;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index c0d4690..e066742 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -29,7 +29,7 @@ import javax.management.ObjectName;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
  import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
  import org.apache.hadoop.hive.llap.daemon.QueryFailedHandler;
  import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
@@ -302,7 +302,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
      try {
        // Cache settings will need to be setup in llap-daemon-site.xml - since the daemons don't read hive-site.xml
        // Ideally, these properties should be part of LlapDameonConf rather than HiveConf
- LlapConfiguration daemonConf = new LlapConfiguration();
+ LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
        int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);

        String localDirList = HiveConf.getVar(daemonConf, ConfVars.LLAP_DAEMON_WORK_DIRS);

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
index 80264a0..14657e6 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
  import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
  import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
  import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
@@ -42,7 +41,6 @@ import java.util.List;
  import java.util.Set;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
  import java.util.concurrent.locks.Lock;
  import java.util.concurrent.locks.ReadWriteLock;
  import java.util.concurrent.locks.ReentrantLock;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
index 2c51169..39a1468 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
@@ -119,10 +119,10 @@ public class ShuffleHandler implements AttemptRegistrationListener {

    public static final String SHUFFLE_HANDLER_LOCAL_DIRS = "llap.shuffle.handler.local-dirs";

- public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+ public static final String SHUFFLE_MANAGE_OS_CACHE = "lla[.shuffle.manage.os.cache";
    public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;

- public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+ public static final String SHUFFLE_READAHEAD_BYTES = "llap.shuffle.readahead.bytes";
    public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;

    public static final String SHUFFLE_DIR_WATCHER_ENABLED = "llap.shuffle.dir-watcher.enabled";
@@ -163,40 +163,39 @@ public class ShuffleHandler implements AttemptRegistrationListener {
    public static final String SHUFFLE_PORT_CONFIG_KEY = "llap.shuffle.port";
    public static final int DEFAULT_SHUFFLE_PORT = 15551;

- // TODO Change configs to remove mapreduce references.
    public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
- "mapreduce.shuffle.connection-keep-alive.enable";
+ "llap.shuffle.connection-keep-alive.enable";
    public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;

    public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
- "mapreduce.shuffle.connection-keep-alive.timeout";
+ "llap.shuffle.connection-keep-alive.timeout";
    public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = 5; //seconds

    public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
- "mapreduce.shuffle.mapoutput-info.meta.cache.size";
+ "llap.shuffle.mapoutput-info.meta.cache.size";
    public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
        10000;

    public static final String CONNECTION_CLOSE = "close";

    public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
- "mapreduce.shuffle.ssl.file.buffer.size";
+ "llap.shuffle.ssl.file.buffer.size";

    public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;

- public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections";
+ public static final String MAX_SHUFFLE_CONNECTIONS = "llap.shuffle.max.connections";
    public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit

- public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads";
+ public static final String MAX_SHUFFLE_THREADS = "llap.shuffle.max.threads";
    // 0 implies Netty default of 2 * number of available processors
    public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;

    public static final String SHUFFLE_BUFFER_SIZE =
- "mapreduce.shuffle.transfer.buffer.size";
+ "llap.shuffle.transfer.buffer.size";
    public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;

    public static final String SHUFFLE_TRANSFERTO_ALLOWED =
- "mapreduce.shuffle.transferTo.allowed";
+ "llap.shuffle.transferTo.allowed";
    public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
    public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED =
        false;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/main/resources/llap-cli-log4j2.properties
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/llap-cli-log4j2.properties b/llap-server/src/main/resources/llap-cli-log4j2.properties
new file mode 100644
index 0000000..7542193
--- /dev/null
+++ b/llap-server/src/main/resources/llap-cli-log4j2.properties
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = WARN
+name = LlapCliLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+property.hive.log.level = INFO
+property.hive.root.logger = console
+property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
+property.hive.log.file = hive.log
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+# appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
+appender.console.layout.pattern = %p %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HadoopConf
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.HadoopConf.name = org.apache.hadoop.conf.Configuration
+logger.HadoopConf.level = ERROR
+
+# root logger
+rootLogger.level = ${sys:hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
index deade5f..5fa2bf1 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileContext;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
  import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
  import org.apache.hadoop.service.AbstractService;
  import org.apache.hadoop.service.Service;

http://git-wip-us.apache.org/repos/asf/hive/blob/f8f50ab1/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
index 1cef218..a65bf5c 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
@@ -25,7 +25,7 @@ import com.google.protobuf.ServiceException;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
  import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
  import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
  import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
@@ -39,7 +39,7 @@ public class TestLlapDaemonProtocolServerImpl {

    @Test(timeout = 10000)
    public void test() throws ServiceException, IOException {
- LlapConfiguration daemonConf = new LlapConfiguration();
+ LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
      int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT);
      int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
      ContainerRunner containerRunnerMock = mock(ContainerRunner.class);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 17, '16 at 1:13a
activeFeb 17, '16 at 1:13a
posts1
users1
websitehive.apache.org

1 user in discussion

Gopalv: 1 post

People

Translate

site design / logo © 2021 Grokbase