FAQ
Author: thejas
Date: Fri Jun 6 21:07:26 2014
New Revision: 1601012

URL: http://svn.apache.org/r1601012
Log:
HIVE-7065 : Hive jobs in webhcat run in default mr mode even in Hive on Tez setup (Eugene Koifman via Thejas Nair)

Modified:
     hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml
     hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
     hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.xml Fri Jun 6 21:07:26 2014
@@ -31,6 +31,10 @@
          <value>thrift://localhost:9933</value>
          <description>For Hive CLI to connect to</description>
      </property>
+ <property>
+ <name>hive.execution.engine</name>
+ <value>mr</value><!--tez-->
+ </property>
      <!--
      This enables client side Hive (RDBMS style) security
      <property>

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Fri Jun 6 21:07:26 2014
@@ -131,8 +131,9 @@
    <property>
      <name>templeton.hive.properties</name>
      <value>hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
- <description>Properties to set when running hive. This is expected to be a comma-separated prop=value list.
- If some value is itself a comma-separated list the escape character is '\'</description>
+ <description>Properties to set when running hive (during job sumission). This is expected to
+ be a comma-separated prop=value list. If some value is itself a comma-separated list the
+ escape character is '\'</description>
    </property>

    <property>

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Fri Jun 6 21:07:26 2014
@@ -33,10 +33,10 @@ import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.util.StringUtils;
  import org.apache.hadoop.util.VersionInfo;
  import org.apache.hive.hcatalog.templeton.tool.JobState;
-import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
  import org.apache.hive.hcatalog.templeton.tool.ZooKeeperCleanup;
  import org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage;

@@ -111,7 +111,8 @@ public class AppConfig extends Configura
    /**
     * is a comma separated list of name=value pairs;
     * In case some value is itself a comma-separated list, the comma needs to
- * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}
+ * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}. See other usage
+ * of escape/unescape methods in {@link org.apache.hadoop.util.StringUtils} in webhcat.
     */
    public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
    public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive";
@@ -167,8 +168,49 @@ public class AppConfig extends Configura
        loadOneFileConfig(hadoopConfDir, fname);
      }
      ProxyUserSupport.processProxyuserConfig(this);
+ handleHiveProperties();
      LOG.info(dumpEnvironent());
    }
+ /**
+ * When auto-shipping hive tar (for example when hive query or pig script
+ * is submitted via webhcat), Hive client is launched on some remote node where Hive has not
+ * been installed. We need pass some properties to that client to make sure it connects to the
+ * right Metastore, configures Tez, etc. Here we look for such properties in hive config,
+ * and set a comma-separated list of key values in {@link #HIVE_PROPS_NAME}.
+ * Note that the user may choose to set the same keys in HIVE_PROPS_NAME directly, in which case
+ * those values should take precedence.
+ */
+ private void handleHiveProperties() {
+ HiveConf hiveConf = new HiveConf();//load hive-site.xml from classpath
+ List<String> interestingPropNames = Arrays.asList(
+ "hive.metastore.uris","hive.metastore.sasl.enabled",
+ "hive.metastore.execute.setugi","hive.execution.engine");
+
+ //each items is a "key=value" format
+ List<String> webhcatHiveProps = new ArrayList<String>(hiveProps());
+ for(String interestingPropName : interestingPropNames) {
+ String value = hiveConf.get(interestingPropName);
+ if(value != null) {
+ boolean found = false;
+ for(String whProp : webhcatHiveProps) {
+ if(whProp.startsWith(interestingPropName + "=")) {
+ found = true;
+ break;
+ }
+ }
+ if(!found) {
+ webhcatHiveProps.add(interestingPropName + "=" + value);
+ }
+ }
+ }
+ StringBuilder hiveProps = new StringBuilder();
+ for(String whProp : webhcatHiveProps) {
+ //make sure to escape separator char in prop values
+ hiveProps.append(hiveProps.length() > 0 ? "," : "").append(StringUtils.escapeString(whProp));
+ }
+ set(HIVE_PROPS_NAME, hiveProps.toString());
+ }
+
    private static void logConfigLoadAttempt(String path) {
      LOG.info("Attempting to load config file: " + path);
    }
@@ -198,7 +240,8 @@ public class AppConfig extends Configura
        }
      });
      for(Map.Entry<String, String> entry : configVals) {
- sb.append(entry.getKey()).append('=').append(entry.getValue()).append('\n');
+ //use get() to make sure variable substitution works
+ sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
      }
      return sb.toString();
    }
@@ -266,7 +309,7 @@ public class AppConfig extends Configura
      //since raw data was (possibly) escaped to make split work,
      //now need to remove escape chars so they don't interfere with downstream processing
      for(int i = 0; i < props.length; i++) {
- props[i] = TempletonUtils.unEscape(props[i]);
+ props[i] = StringUtils.unEscapeString(props[i]);
      }
      return Arrays.asList(props);
    }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Fri Jun 6 21:07:26 2014
@@ -81,7 +81,7 @@ public class LaunchMapper extends Mapper
      if(conf.get(PigConstants.PIG_OPTS) != null) {
        StringBuilder pigOpts = new StringBuilder();
        for(String prop : StringUtils.split(conf.get(PigConstants.PIG_OPTS))) {
- pigOpts.append("-D").append(TempletonUtils.unEscape(prop)).append(" ");
+ pigOpts.append("-D").append(StringUtils.unEscapeString(prop)).append(" ");
        }
        env.put(PigConstants.PIG_OPTS, pigOpts.toString());
      }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Fri Jun 6 21:07:26 2014
@@ -347,8 +347,4 @@ public class TempletonUtils {
        args.add("call");
      }
    }
-
- public static String unEscape(String input) {
- return StringEscapeUtils.unescapeJava(input);
- }
  }

Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1601012&r1=1601011&r2=1601012&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Fri Jun 6 21:07:26 2014
@@ -306,7 +306,7 @@ public class TestTempletonUtils {
      String[] newProps = StringUtils.split(input.toString());
      for(int i = 0; i < newProps.length; i++) {
        Assert.assertEquals("Pre/post split values don't match",
- TempletonUtils.unEscape(props[i]), TempletonUtils.unEscape(newProps[i]));
+ StringUtils.unEscapeString(props[i]), StringUtils.unEscapeString(newProps[i]));
      }
    }
  }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 6, '14 at 9:07p
activeJun 6, '14 at 9:07p
posts1
users1
websitehive.apache.org

1 user in discussion

Thejas: 1 post

People

Translate

site design / logo © 2021 Grokbase