FAQ
Author: thejas
Date: Tue Oct 1 00:50:12 2013
New Revision: 1527835

URL: http://svn.apache.org/r1527835
Log:
HIVE-5035: [WebHCat] Hardening parameters for Windows (Daniel Dai via Thejas Nair)

Modified:
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1527835&r1=1527834&r2=1527835&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Oct 1 00:50:12 2013
@@ -75,16 +75,18 @@ public class HiveDelegator extends Launc

        for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
          args.add("--hiveconf");
- args.add(prop);
+ args.add(TempletonUtils.quoteForWindows(prop));
        }
        for (String prop : defines) {
          args.add("--hiveconf");
- args.add(prop);
+ args.add(TempletonUtils.quoteForWindows(prop));
+ }
+ for (String hiveArg : hiveArgs) {
+ args.add(TempletonUtils.quoteForWindows(hiveArg));
        }
- args.addAll(hiveArgs);
        if (TempletonUtils.isset(execute)) {
          args.add("-e");
- args.add(execute);
+ args.add(TempletonUtils.quoteForWindows(execute));
        } else if (TempletonUtils.isset(srcFile)) {
          args.add("-f");
          args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs)

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1527835&r1=1527834&r2=1527835&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Oct 1 00:50:12 2013
@@ -74,22 +74,27 @@ public class JarDelegator extends Launch
        if (TempletonUtils.isset(mainClass))
          args.add(mainClass);
        if (TempletonUtils.isset(libjars)) {
+ String libjarsListAsString =
+ TempletonUtils.hadoopFsListAsString(libjars, appConf, runAs);
          args.add("-libjars");
- args.add(TempletonUtils.hadoopFsListAsString(libjars, appConf,
- runAs));
+ args.add(TempletonUtils.quoteForWindows(libjarsListAsString));
        }
        if (TempletonUtils.isset(files)) {
+ String filesListAsString =
+ TempletonUtils.hadoopFsListAsString(files, appConf, runAs);
          args.add("-files");
- args.add(TempletonUtils.hadoopFsListAsString(files, appConf,
- runAs));
+ args.add(TempletonUtils.quoteForWindows(filesListAsString));
        }
        //the token file location comes after mainClass, as a -Dprop=val
        args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);

- for (String d : defines)
- args.add("-D" + d);
-
- args.addAll(jarArgs);
+ for (String d : defines) {
+ args.add("-D");
+ TempletonUtils.quoteForWindows(d);
+ }
+ for (String arg : jarArgs) {
+ args.add(TempletonUtils.quoteForWindows(arg));
+ }
      } catch (FileNotFoundException e) {
        throw new BadParam(e.getMessage());
      } catch (URISyntaxException e) {

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1527835&r1=1527834&r2=1527835&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Oct 1 00:50:12 2013
@@ -78,10 +78,12 @@ public class PigDelegator extends Launch
        //the token file location should be first argument of pig
        args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);

- args.addAll(pigArgs);
+ for (String pigArg : pigArgs) {
+ args.add(TempletonUtils.quoteForWindows(pigArg));
+ }
        if (TempletonUtils.isset(execute)) {
          args.add("-execute");
- args.add(execute);
+ args.add(TempletonUtils.quoteForWindows(execute));
        } else if (TempletonUtils.isset(srcFile)) {
          args.add("-file");
          args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs)

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1527835&r1=1527834&r2=1527835&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Oct 1 00:50:12 2013
@@ -587,7 +587,9 @@ public class Server {
                        @FormParam("output") String output,
                        @FormParam("mapper") String mapper,
                        @FormParam("reducer") String reducer,
- @FormParam("file") List<String> files,
+ @FormParam("combiner") String combiner,
+ @FormParam("file") List<String> fileList,
+ @FormParam("files") String files,
                        @FormParam("define") List<String> defines,
                        @FormParam("cmdenv") List<String> cmdenvs,
                        @FormParam("arg") List<String> args,
@@ -607,6 +609,8 @@ public class Server {
      userArgs.put("output", output);
      userArgs.put("mapper", mapper);
      userArgs.put("reducer", reducer);
+ userArgs.put("combiner", combiner);
+ userArgs.put("file", fileList);
      userArgs.put("files", files);
      userArgs.put("define", defines);
      userArgs.put("cmdenv", cmdenvs);
@@ -617,8 +621,8 @@ public class Server {
      checkEnableLogPrerequisite(enablelog, statusdir);

      StreamingDelegator d = new StreamingDelegator(appConf);
- return d.run(getDoAsUser(), userArgs, inputs, output, mapper, reducer,
- files, defines, cmdenvs, args,
+ return d.run(getDoAsUser(), userArgs, inputs, output, mapper, reducer, combiner,
+ fileList, files, defines, cmdenvs, args,
        statusdir, callback, getCompletedUrl(), enablelog, JobType.STREAMING);
    }


Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java?rev=1527835&r1=1527834&r2=1527835&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java Tue Oct 1 00:50:12 2013
@@ -24,6 +24,7 @@ import java.util.List;
  import java.util.Map;

  import org.apache.commons.exec.ExecuteException;
+import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;

  /**
   * Submit a streaming job to the MapReduce queue. Really just a front
@@ -38,8 +39,9 @@ public class StreamingDelegator extends

    public EnqueueBean run(String user, Map<String, Object> userArgs,
                 List<String> inputs, String output,
- String mapper, String reducer,
- List<String> files, List<String> defines,
+ String mapper, String reducer, String combiner,
+ List<String> fileList,
+ String files, List<String> defines,
                 List<String> cmdenvs,
                 List<String> jarArgs,
                 String statusdir,
@@ -49,13 +51,13 @@ public class StreamingDelegator extends
                 JobType jobType)
      throws NotAuthorizedException, BadParam, BusyException, QueueException,
      ExecuteException, IOException, InterruptedException {
- List<String> args = makeArgs(inputs, output, mapper, reducer,
- files, defines, cmdenvs, jarArgs);
+ List<String> args = makeArgs(inputs, output, mapper, reducer, combiner,
+ fileList, cmdenvs, jarArgs);

      JarDelegator d = new JarDelegator(appConf);
      return d.run(user, userArgs,
        appConf.streamingJar(), null,
- null, null, args, defines,
+ null, files, args, defines,
        statusdir, callback, completedUrl, enableLog, jobType);
    }

@@ -63,10 +65,12 @@ public class StreamingDelegator extends
                    String output,
                    String mapper,
                    String reducer,
- List<String> files,
- List<String> defines,
+ String combiner,
+ List<String> fileList,
                    List<String> cmdenvs,
- List<String> jarArgs) {
+ List<String> jarArgs)
+ throws BadParam
+ {
      ArrayList<String> args = new ArrayList<String>();
      for (String input : inputs) {
        args.add("-input");
@@ -79,13 +83,24 @@ public class StreamingDelegator extends
      args.add("-reducer");
      args.add(reducer);

- for (String f : files)
- args.add("-file" + f);
- for (String d : defines)
- args.add("-D" + d);
- for (String e : cmdenvs)
- args.add("-cmdenv" + e);
- args.addAll(jarArgs);
+ if (TempletonUtils.isset(combiner)) {
+ args.add("-combiner");
+ args.add(combiner);
+ }
+
+ for (String f : fileList) {
+ args.add("-file");
+ args.add(f);
+ }
+
+ for (String e : cmdenvs) {
+ args.add("-cmdenv");
+ args.add(TempletonUtils.quoteForWindows(e));
+ }
+
+ for (String arg : jarArgs) {
+ args.add(TempletonUtils.quoteForWindows(arg));
+ }

      return args;
    }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1527835&r1=1527834&r2=1527835&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Tue Oct 1 00:50:12 2013
@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
  import java.net.URL;
  import java.net.URLConnection;
  import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
  import java.util.Collection;
  import java.util.HashMap;
  import java.util.List;
@@ -39,8 +40,10 @@ import org.apache.hadoop.conf.Configurat
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
  import org.apache.hadoop.util.StringUtils;
  import org.apache.hive.hcatalog.templeton.UgiFactory;
+import org.apache.hive.hcatalog.templeton.BadParam;

  /**
   * General utility methods.
@@ -296,4 +299,46 @@ public class TempletonUtils {

      return env;
    }
+
+ // Add double quotes around the given input parameter if it is not already
+ // quoted. Quotes are not allowed in the middle of the parameter, and
+ // BadParam exception is thrown if this is the case.
+ //
+ // This method should be used to escape parameters before they get passed to
+ // Windows cmd scripts (specifically, special characters like a comma or an
+ // equal sign might be lost as part of the cmd script processing if not
+ // under quotes).
+ public static String quoteForWindows(String param) throws BadParam {
+ if (Shell.WINDOWS) {
+ if (param != null && param.length() > 0) {
+ String nonQuotedPart = param;
+ boolean addQuotes = true;
+ if (param.charAt(0) == '\"' && param.charAt(param.length() - 1) == '\"') {
+ if (param.length() < 2)
+ throw new BadParam("Passed in parameter is incorrectly quoted: " + param);
+
+ addQuotes = false;
+ nonQuotedPart = param.substring(1, param.length() - 1);
+ }
+
+ // If we have any quotes other then the outside quotes, throw
+ if (nonQuotedPart.contains("\"")) {
+ throw new BadParam("Passed in parameter is incorrectly quoted: " + param);
+ }
+
+ if (addQuotes) {
+ param = '\"' + param + '\"';
+ }
+ }
+ }
+ return param;
+ }
+
+ public static void addCmdForWindows(ArrayList<String> args) {
+ if(Shell.WINDOWS){
+ args.add("cmd");
+ args.add("/c");
+ args.add("call");
+ }
+ }
  }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 1, '13 at 12:50a
activeOct 1, '13 at 12:50a
posts1
users1
websitehive.apache.org

1 user in discussion

Thejas: 1 post

People

Translate

site design / logo © 2021 Grokbase