Grokbase Groups Hive commits May 2015
FAQ
Repository: hive
Updated Branches:
   refs/heads/master 7ad9b1489 -> fa2dd8391


HIVE-10858 WebHCat specific resources should be added to HADOOP_CLASSPATH first(Eugene Koifman, Reviewed by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fa2dd839
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fa2dd839
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fa2dd839

Branch: refs/heads/master
Commit: fa2dd83913b2464d81504a97a9244187889a3c9f
Parents: 7ad9b14
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri May 29 15:28:21 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Fri May 29 15:28:21 2015 -0700

----------------------------------------------------------------------
  .../templeton/tool/JobSubmissionConstants.java | 1 +
  .../hcatalog/templeton/tool/LaunchMapper.java | 27 +++++++++++++-------
  2 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/fa2dd839/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
index 1d560b6..d3dc3f7 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
@@ -55,6 +55,7 @@ public interface JobSubmissionConstants {
    public static final String MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER =
      "__MR_JOB_TAGS_OPTION=MR_JOB_TAGS_JOBID__";

+ public static final String HADOOP_CLASSPATH = "HADOOP_CLASSPATH";
    /**
     * constants needed for Pig job submission
     * The string values here are what Pig expects to see in it's environment

http://git-wip-us.apache.org/repos/asf/hive/blob/fa2dd839/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
index 3edd449..91fe247 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
@@ -107,12 +107,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
        }
        jdbcJars.setLength(jdbcJars.length() - 1);
        //this makes the jars available to Sqoop client
- if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) {
- env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + jdbcJars.toString());
- }
- else {
- env.put("HADOOP_CLASSPATH", jdbcJars.toString());
- }
+ prependPathToVariable(HADOOP_CLASSPATH, env, jdbcJars.toString());
      }
    }
    private static void handleHadoopClasspathExtras(Configuration conf, Map<String, String> env)
@@ -134,11 +129,25 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
        paths.append(File.pathSeparator);
      }
      paths.setLength(paths.length() - 1);
- if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) {
- env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + paths);
+ prependPathToVariable(HADOOP_CLASSPATH, env, paths.toString());
+ }
+ /**
+ * Ensures that {@code paths} are prepended to {@code pathVarName} and made available to forked child
+ * process.
+ * @param paths properly separated list of paths
+ */
+ private static void prependPathToVariable(String pathVarName, Map<String, String> env, String paths) {
+ if(!TempletonUtils.isset(pathVarName) || !TempletonUtils.isset(paths) || env == null) {
+ return;
+ }
+ if(TempletonUtils.isset(env.get(pathVarName))) {
+ env.put(pathVarName, paths + File.pathSeparator + env.get(pathVarName));
+ }
+ else if(TempletonUtils.isset(System.getenv(pathVarName))) {
+ env.put(pathVarName, paths + File.pathSeparator + System.getenv(pathVarName));
      }
      else {
- env.put("HADOOP_CLASSPATH", paths.toString());
+ env.put(pathVarName, paths);
      }
    }
    protected Process startJob(Context context, String user, String overrideClasspath)

Search Discussions

Discussion Posts

Previous

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 29, '15 at 10:22p
activeMay 29, '15 at 10:28p
posts2
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 2 posts

People

Translate

site design / logo © 2021 Grokbase