FAQ
Author: thejas
Date: Tue Oct 1 00:54:17 2013
New Revision: 1527840

URL: http://svn.apache.org/r1527840
Log:
HIVE-5066: [WebHCat] Other code fixes for Windows (Daniel Dai via Thejas Nair)

Modified:
     hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
     hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
     hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java
     hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Tue Oct 1 00:54:17 2013
@@ -80,32 +80,38 @@
    </property>

    <property>
+ <name>templeton.python</name>
+ <value>${env.PYTHON_CMD}</value>
+ <description>The path to the python executable.</description>
+ </property>
+
+ <property>
      <name>templeton.pig.archive</name>
- <value>hdfs:///apps/templeton/pig-0.10.1.tar.gz</value>
+ <value></value>
      <description>The path to the Pig archive.</description>
    </property>

    <property>
      <name>templeton.pig.path</name>
- <value>pig-0.10.1.tar.gz/pig-0.10.1/bin/pig</value>
+ <value>pig-0.11.1.tar.gz/pig-0.11.1/bin/pig</value>
      <description>The path to the Pig executable.</description>
    </property>

    <property>
      <name>templeton.hcat</name>
- <value>${env.HCAT_PREFIX}/bin/hcat</value>
+ <value>${env.HCAT_PREFIX}/bin/hcat.py</value>
      <description>The path to the hcatalog executable.</description>
    </property>

    <property>
      <name>templeton.hive.archive</name>
- <value>hdfs:///apps/templeton/hive-0.10.0.tar.gz</value>
+ <value></value>
      <description>The path to the Hive archive.</description>
    </property>

    <property>
      <name>templeton.hive.path</name>
- <value>hive-0.10.0.tar.gz/hive-0.10.0/bin/hive</value>
+ <value>hive-0.11.0.tar.gz/hive-0.11.0/bin/hive</value>
      <description>The path to the Hive executable.</description>
    </property>


Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Tue Oct 1 00:54:17 2013
@@ -87,6 +87,7 @@ public class AppConfig extends Configura
    public static final String HADOOP_NAME = "templeton.hadoop";
    public static final String HADOOP_CONF_DIR = "templeton.hadoop.conf.dir";
    public static final String HCAT_NAME = "templeton.hcat";
+ public static final String PYTHON_NAME = "templeton.python";
    public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
    public static final String HIVE_PATH_NAME = "templeton.hive.path";
    public static final String HIVE_PROPS_NAME = "templeton.hive.properties";
@@ -181,6 +182,7 @@ public class AppConfig extends Configura
    public String hadoopQueueName() { return get(HADOOP_QUEUE_NAME); }
    public String clusterHadoop() { return get(HADOOP_NAME); }
    public String clusterHcat() { return get(HCAT_NAME); }
+ public String clusterPython() { return get(PYTHON_NAME); }
    public String pigPath() { return get(PIG_PATH_NAME); }
    public String pigArchive() { return get(PIG_ARCHIVE_NAME); }
    public String hivePath() { return get(HIVE_PATH_NAME); }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java Tue Oct 1 00:54:17 2013
@@ -18,12 +18,18 @@
   */
  package org.apache.hive.hcatalog.templeton;

+import java.io.BufferedReader;
  import java.io.ByteArrayOutputStream;
  import java.io.File;
  import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
+import java.util.Map.Entry;
  import java.util.concurrent.Semaphore;

  import org.apache.commons.exec.CommandLine;
@@ -33,6 +39,38 @@ import org.apache.commons.exec.ExecuteWa
  import org.apache.commons.exec.PumpStreamHandler;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
+
+class StreamOutputWriter extends Thread
+{
+ InputStream is;
+ String type;
+ PrintWriter out;
+
+ StreamOutputWriter(InputStream is, String type, OutputStream outStream)
+ {
+ this.is = is;
+ this.type = type;
+ this.out = new PrintWriter(outStream, true);
+ }
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ BufferedReader br =
+ new BufferedReader(new InputStreamReader(is));
+ String line = null;
+ while ( (line = br.readLine()) != null){
+ out.println(line);
+ }
+ } catch (IOException ioe)
+ {
+ ioe.printStackTrace();
+ }
+ }
+}

  /**
   * Execute a local program. This is a singleton service that will
@@ -45,6 +83,9 @@ public class ExecServiceImpl implements

    private static volatile ExecServiceImpl theSingleton;

+ /** Windows CreateProcess synchronization object */
+ private static final Object WindowsProcessLaunchLock = new Object();
+
    /**
     * Retrieve the singleton.
     */
@@ -133,7 +174,54 @@ public class ExecServiceImpl implements

      LOG.info("Running: " + cmd);
      ExecBean res = new ExecBean();
- res.exitcode = executor.execute(cmd, execEnv(env));
+
+ if(Shell.WINDOWS){
+ //The default executor is sometimes causing failure on windows. hcat
+ // command sometimes returns non zero exit status with it. It seems
+ // to hit some race conditions on windows.
+ env = execEnv(env);
+ String[] envVals = new String[env.size()];
+ int i=0;
+ for( Entry<String, String> kv : env.entrySet()){
+ envVals[i++] = kv.getKey() + "=" + kv.getValue();
+ LOG.info("Setting " + kv.getKey() + "=" + kv.getValue());
+ }
+
+ Process proc;
+ synchronized (WindowsProcessLaunchLock) {
+ // To workaround the race condition issue with child processes
+ // inheriting unintended handles during process launch that can
+ // lead to hangs on reading output and error streams, we
+ // serialize process creation. More info available at:
+ // http://support.microsoft.com/kb/315939
+ proc = Runtime.getRuntime().exec(cmd.toStrings(), envVals);
+ }
+
+ //consume stderr
+ StreamOutputWriter errorGobbler = new
+ StreamOutputWriter(proc.getErrorStream(), "ERROR", errStream);
+
+ //consume stdout
+ StreamOutputWriter outputGobbler = new
+ StreamOutputWriter(proc.getInputStream(), "OUTPUT", outStream);
+
+ //start collecting input streams
+ errorGobbler.start();
+ outputGobbler.start();
+ //execute
+ try{
+ res.exitcode = proc.waitFor();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ //flush
+ errorGobbler.out.flush();
+ outputGobbler.out.flush();
+ }
+ else {
+ res.exitcode = executor.execute(cmd, execEnv(env));
+ }
+
      String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME);
      res.stdout = outStream.toString(enc);
      res.stderr = errStream.toString(enc);

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java Tue Oct 1 00:54:17 2013
@@ -67,7 +67,11 @@ public class HcatDelegator extends Launc
        Map<String, String> env = TempletonUtils.hadoopUserEnv(user, cp);
        proxy.addEnv(env);
        proxy.addArgs(args);
- return execService.run(appConf.clusterHcat(), args, env);
+ if (appConf.clusterHcat().toLowerCase().endsWith(".py")) {
+ return execService.run(appConf.clusterPython(), args, env);
+ } else {
+ return execService.run(appConf.clusterHcat(), args, env);
+ }
      } catch (InterruptedException e) {
        throw new IOException(e);
      } finally {
@@ -79,8 +83,12 @@ public class HcatDelegator extends Launc
    private List<String> makeArgs(String exec, boolean format,
                    String group, String permissions) {
      ArrayList<String> args = new ArrayList<String>();
+ if (appConf.clusterHcat().toLowerCase().endsWith(".py")) {
+ // hcat.py will become the first argument pass to command "python"
+ args.add(appConf.clusterHcat());
+ }
      args.add("-e");
- args.add(exec);
+ args.add('"' + exec + '"');
      if (TempletonUtils.isset(group)) {
        args.add("-g");
        args.add(group);

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Oct 1 00:54:17 2013
@@ -64,6 +64,7 @@ public class HiveDelegator extends Launc
      try {
        args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog));
        args.add("--");
+ TempletonUtils.addCmdForWindows(args);
        args.add(appConf.hivePath());

        args.add("--service");
@@ -122,8 +123,11 @@ public class HiveDelegator extends Launc
      args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
                  enablelog, JobType.HIVE));

- args.add("-archives");
- args.add(appConf.hiveArchive());
+ if (appConf.hiveArchive() != null && !appConf.hiveArchive().equals(""))
+ {
+ args.add("-archives");
+ args.add(appConf.hiveArchive());
+ }

      return args;
    }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Oct 1 00:54:17 2013
@@ -68,6 +68,7 @@ public class JarDelegator extends Launch
        args.addAll(makeLauncherArgs(appConf, statusdir,
          completedUrl, allFiles, enablelog, jobType));
        args.add("--");
+ TempletonUtils.addCmdForWindows(args);
        args.add(appConf.clusterHadoop());
        args.add("jar");
        args.add(TempletonUtils.hadoopFsPath(jar, appConf, runAs).getName());

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Oct 1 00:54:17 2013
@@ -70,10 +70,14 @@ public class PigDelegator extends Launch
        }

        args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG));
- args.add("-archives");
- args.add(appConf.pigArchive());
+ if (appConf.pigArchive() != null && !appConf.pigArchive().equals(""))
+ {
+ args.add("-archives");
+ args.add(appConf.pigArchive());
+ }

        args.add("--");
+ TempletonUtils.addCmdForWindows(args);
        args.add(appConf.pigPath());
        //the token file location should be first argument of pig
        args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSCleanup.java Tue Oct 1 00:54:17 2013
@@ -98,7 +98,7 @@ public class HDFSCleanup extends Thread
          // cycle fails, it'll try again on the next cycle.
          try {
            if (fs == null) {
- fs = FileSystem.get(appConf);
+ fs = new Path(storage_root).getFileSystem(appConf);
            }
            checkFiles(fs);
          } catch (Exception e) {

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java Tue Oct 1 00:54:17 2013
@@ -210,7 +210,7 @@ public class HDFSStorage implements Temp
    public void openStorage(Configuration config) throws IOException {
      storage_root = config.get(TempletonStorage.STORAGE_ROOT);
      if (fs == null) {
- fs = FileSystem.get(config);
+ fs = new Path(storage_root).getFileSystem(config);
      }
    }


Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Tue Oct 1 00:54:17 2013
@@ -48,12 +48,15 @@ import org.apache.hadoop.mapred.JobConf;
  import org.apache.hadoop.mapreduce.Job;
  import org.apache.hadoop.mapreduce.JobID;
  import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Mapper.Context;
  import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
  import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
  import org.apache.hadoop.security.UserGroupInformation;
  import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
  import org.apache.hadoop.util.Tool;
  import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.templeton.BadParam;
  import org.apache.hive.hcatalog.templeton.LauncherDelegator;

  /**
@@ -104,6 +107,9 @@ public class TempletonControllerJob exte

        ArrayList<String> removeEnv = new ArrayList<String>();
        removeEnv.add("HADOOP_ROOT_LOGGER");
+ removeEnv.add("hadoop-command");
+ removeEnv.add("CLASS");
+ removeEnv.add("mapredcommand");
        Map<String, String> env = TempletonUtils.hadoopUserEnv(user,
          overrideClasspath);
        List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
@@ -112,7 +118,15 @@ public class TempletonControllerJob exte

        if (tokenFile != null) {
          //Token is available, so replace the placeholder
+ tokenFile = tokenFile.replaceAll("\"", "");
          String tokenArg = "mapreduce.job.credentials.binary=" + tokenFile;
+ if (Shell.WINDOWS) {
+ try {
+ tokenArg = TempletonUtils.quoteForWindows(tokenArg);
+ } catch (BadParam e) {
+ throw new IOException("cannot pass " + tokenFile + " to mapreduce.job.credentials.binary", e);
+ }
+ }
          for(int i=0; i<jarArgsList.size(); i++){
            String newArg =
              jarArgsList.get(i).replace(TOKEN_FILE_ARG_PLACEHOLDER, tokenArg);
@@ -211,9 +225,9 @@ public class TempletonControllerJob exte
        pool.execute(w);
      }

- private KeepAlive startCounterKeepAlive(ExecutorService pool, Context cnt)
+ private KeepAlive startCounterKeepAlive(ExecutorService pool, Context context)
        throws IOException {
- KeepAlive k = new KeepAlive(cnt);
+ KeepAlive k = new KeepAlive(context);
        pool.execute(k);
        return k;
      }
@@ -297,20 +311,25 @@ public class TempletonControllerJob exte
      }
    }

- private static class KeepAlive implements Runnable {
- private final Mapper.Context cnt;
- private volatile boolean sendReport;
+ public static class KeepAlive implements Runnable {
+ private Context context;
+ public boolean sendReport;

- public KeepAlive(Mapper.Context cnt) {
- this.cnt = cnt;
+ public KeepAlive(Context context)
+ {
        this.sendReport = true;
+ this.context = context;
      }

      @Override
      public void run() {
        try {
          while (sendReport) {
- cnt.progress();
+ // Periodically report progress on the Context object
+ // to prevent TaskTracker from killing the Templeton
+ // Controller task
+ context.progress();
+ System.err.println("KeepAlive Heart beat");
            Thread.sleep(KEEP_ALIVE_MSEC);
          }
        } catch (InterruptedException e) {

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java Tue Oct 1 00:54:17 2013
@@ -22,12 +22,16 @@ import java.io.IOException;
  import java.util.List;
  import java.util.Map;

+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
  /**
   * Execute a local program. This is a singleton service that will
   * execute a programs on the local box.
   */
  public class TrivialExecService {
    private static volatile TrivialExecService theSingleton;
+ private static final Log LOG = LogFactory.getLog(TrivialExecService.class);

    /**
     * Retrieve the singleton.
@@ -41,11 +45,7 @@ public class TrivialExecService {
    public Process run(List<String> cmd, List<String> removeEnv,
               Map<String, String> environmentVariables)
      throws IOException {
- System.err.println("templeton: starting " + cmd);
- System.err.print("With environment variables: ");
- for (Map.Entry<String, String> keyVal : environmentVariables.entrySet()) {
- System.err.println(keyVal.getKey() + "=" + keyVal.getValue());
- }
+ logDebugCmd(cmd, environmentVariables);
      ProcessBuilder pb = new ProcessBuilder(cmd);
      for (String key : removeEnv)
        pb.environment().remove(key);
@@ -53,4 +53,20 @@ public class TrivialExecService {
      return pb.start();
    }

+ private void logDebugCmd(List<String> cmd,
+ Map<String, String> environmentVariables) {
+ if(!LOG.isDebugEnabled()){
+ return;
+ }
+ LOG.debug("starting " + cmd);
+ LOG.debug("With environment variables: " );
+ for(Map.Entry<String, String> keyVal : environmentVariables.entrySet()){
+ LOG.debug(keyVal.getKey() + "=" + keyVal.getValue());
+ }
+ LOG.debug("With environment variables already set: " );
+ Map<String, String> env = System.getenv();
+ for (String envName : env.keySet()) {
+ LOG.debug(envName + "=" + env.get(envName));
+ }
+ }
  }

Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestServer.java Tue Oct 1 00:54:17 2013
@@ -31,7 +31,7 @@ public class TestServer extends TestCase
    MockServer server;

    public void setUp() {
- new Main(null); // Initialize the config
+ new Main(new String[]{}); // Initialize the config
      server = new MockServer();
    }


Modified: hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1527840&r1=1527839&r2=1527840&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Tue Oct 1 00:54:17 2013
@@ -159,6 +159,12 @@ public class TestTempletonUtils {
    @Test
    public void testHadoopFsFilename() {
      try {
+ String tmpFileName1 = "/tmp/testHadoopFsListAsArray1";
+ String tmpFileName2 = "/tmp/testHadoopFsListAsArray2";
+ File tmpFile1 = new File(tmpFileName1);
+ File tmpFile2 = new File(tmpFileName2);
+ tmpFile1.createNewFile();
+ tmpFile2.createNewFile();
        Assert.assertEquals(null, TempletonUtils.hadoopFsFilename(null, null, null));
        Assert.assertEquals(null,
          TempletonUtils.hadoopFsFilename(tmpFile.toURI().toString(), null, null));
@@ -188,14 +194,22 @@ public class TestTempletonUtils {
    @Test
    public void testHadoopFsListAsArray() {
      try {
+ String tmpFileName1 = "/tmp/testHadoopFsListAsArray1";
+ String tmpFileName2 = "/tmp/testHadoopFsListAsArray2";
+ File tmpFile1 = new File(tmpFileName1);
+ File tmpFile2 = new File(tmpFileName2);
+ tmpFile1.createNewFile();
+ tmpFile2.createNewFile();
        Assert.assertTrue(TempletonUtils.hadoopFsListAsArray(null, null, null) == null);
- Assert.assertTrue(TempletonUtils.hadoopFsListAsArray(
- tmpFile.toURI().toString() + "," + usrFile.toString(), null, null) == null);
- String[] tmp2 = TempletonUtils.hadoopFsListAsArray(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
- new Configuration(), null);
- Assert.assertEquals(tmpFile.toURI().toString(), tmp2[0]);
- Assert.assertEquals(usrFile.toURI().toString(), tmp2[1]);
+ Assert.assertTrue(TempletonUtils.hadoopFsListAsArray(tmpFileName1 + "," + tmpFileName2,
+ null, null) == null);
+ String[] tmp2
+ = TempletonUtils.hadoopFsListAsArray(tmpFileName1 + "," + tmpFileName2,
+ new Configuration(), null);
+ Assert.assertEquals("file:" + tmpFileName1, tmp2[0]);
+ Assert.assertEquals("file:" + tmpFileName2, tmp2[1]);
+ tmpFile1.delete();
+ tmpFile2.delete();
      } catch (FileNotFoundException e) {
        Assert.fail("Couldn't find name for " + tmpFile.toURI().toString());
      } catch (Exception e) {
@@ -218,15 +232,18 @@ public class TestTempletonUtils {
    @Test
    public void testHadoopFsListAsString() {
      try {
+ String tmpFileName1 = "/tmp/testHadoopFsListAsString1";
+ String tmpFileName2 = "/tmp/testHadoopFsListAsString2";
+ File tmpFile1 = new File(tmpFileName1);
+ File tmpFile2 = new File(tmpFileName2);
+ tmpFile1.createNewFile();
+ tmpFile2.createNewFile();
        Assert.assertTrue(TempletonUtils.hadoopFsListAsString(null, null, null) == null);
- Assert.assertTrue(TempletonUtils.hadoopFsListAsString(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
+ Assert.assertTrue(TempletonUtils.hadoopFsListAsString("/tmp,/usr",
          null, null) == null);
- Assert.assertEquals(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
- TempletonUtils.hadoopFsListAsString(
- tmpFile.toURI().toString() + "," + usrFile.toURI().toString(),
- new Configuration(), null));
+ Assert.assertEquals("file:" + tmpFileName1 + ",file:" + tmpFileName2,
+ TempletonUtils.hadoopFsListAsString
+ (tmpFileName1 + "," + tmpFileName2, new Configuration(), null));
      } catch (FileNotFoundException e) {
        Assert.fail("Couldn't find name for " + tmpFile.toURI().toString());
      } catch (Exception e) {

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 1, '13 at 12:54a
activeOct 1, '13 at 12:54a
posts1
users1
websitehive.apache.org

1 user in discussion

Thejas: 1 post

People

Translate

site design / logo © 2021 Grokbase