FAQ
Author: thejas
Date: Wed Apr 2 21:18:08 2014
New Revision: 1584196

URL: http://svn.apache.org/r1584196
Log:
HIVE-6780 : Set tez credential file property along with MR conf property for Tez jobs (Eugene Koifman via Thejas Nair)

Modified:
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Wed Apr 2 21:18:08 2014
@@ -27,6 +27,7 @@ import java.util.List;
  import java.util.Map;

  import org.apache.commons.exec.ExecuteException;
+import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
  import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
  import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;

@@ -76,6 +77,11 @@ public class HiveDelegator extends Launc
        args.add("--hiveconf");
        args.add(TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);

+ //this is needed specifcally for Hive on Tez (in addition to
+ //JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER)
+ args.add("--hiveconf");
+ args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ);
+
        for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
          args.add("--hiveconf");
          args.add(TempletonUtils.quoteForWindows(prop));

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Wed Apr 2 21:18:08 2014
@@ -41,6 +41,8 @@ public interface JobSubmissionConstants
     */
    public static final String TOKEN_FILE_ARG_PLACEHOLDER =
      "__MR_JOB_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION__";
+ public static final String TOKEN_FILE_ARG_PLACEHOLDER_TEZ =
+ "__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__";
    /**
     * constants needed for Pig job submission
     */

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Wed Apr 2 21:18:08 2014
@@ -100,41 +100,47 @@ public class LaunchMapper extends Mapper
      Map<String, String> env = TempletonUtils.hadoopUserEnv(user, overrideClasspath);
      handlePigEnvVars(conf, env);
      List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
- String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
-
+ handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
+ handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
+ boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
+ false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
+ return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
+ }

+ /**
+ * Replace placeholder with actual "prop=file". This is done multiple times (possibly) since
+ * Tez and MR use different property names
+ */
+ private static void handleTokenFile(List<String> jarArgsList, String tokenPlaceHolder, String tokenProperty) throws IOException {
+ String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
      if (tokenFile != null) {
        //Token is available, so replace the placeholder
        tokenFile = tokenFile.replaceAll("\"", "");
- String tokenArg = "mapreduce.job.credentials.binary=" + tokenFile;
+ String tokenArg = tokenProperty + "=" + tokenFile;
        if (Shell.WINDOWS) {
          try {
            tokenArg = TempletonUtils.quoteForWindows(tokenArg);
          } catch (BadParam e) {
- String msg = "cannot pass " + tokenFile + " to mapreduce.job.credentials.binary";
+ String msg = "cannot pass " + tokenFile + " to " + tokenProperty;
            LOG.error(msg, e);
            throw new IOException(msg, e);
          }
        }
        for(int i=0; i<jarArgsList.size(); i++){
          String newArg =
- jarArgsList.get(i).replace(TOKEN_FILE_ARG_PLACEHOLDER, tokenArg);
+ jarArgsList.get(i).replace(tokenPlaceHolder, tokenArg);
          jarArgsList.set(i, newArg);
        }
-
      }else{
        //No token, so remove the placeholder arg
        Iterator<String> it = jarArgsList.iterator();
        while(it.hasNext()){
          String arg = it.next();
- if(arg.contains(TOKEN_FILE_ARG_PLACEHOLDER)){
+ if(arg.contains(tokenPlaceHolder)){
            it.remove();
          }
        }
      }
- boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
- false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
- return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
    }

    private void copyLocal(String var, Configuration conf) throws IOException {

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 2, '14 at 9:18p
activeApr 2, '14 at 9:18p
posts1
users1
websitehive.apache.org

1 user in discussion

Thejas: 1 post

People

Translate

site design / logo © 2021 Grokbase