FAQ
Repository: hive
Updated Branches:
   refs/heads/master fe14a9088 -> 415373bbd


HIVE-11483 : Add encoding and decoding for query string config (Rajat Khandelwal, reviewed by Amareshwari Sriramadasu


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/415373bb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/415373bb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/415373bb

Branch: refs/heads/master
Commit: 415373bbd1c58e7b8f00aa32c0bb7d485d4f91e5
Parents: fe14a90
Author: Rajat Khandelwal <prongs@apache.org>
Authored: Wed Mar 9 14:17:21 2016 +0530
Committer: Amareshwari Sriramadasu <amareshwari@apache.org>
Committed: Wed Mar 9 14:17:21 2016 +0530

----------------------------------------------------------------------
  .../org/apache/hadoop/hive/conf/HiveConf.java | 76 ++++++++++++++++----
  .../apache/hadoop/hive/conf/TestHiveConf.java | 10 +++
  .../java/org/apache/hadoop/hive/ql/Driver.java | 5 +-
  .../hive/ql/exec/errors/TaskLogProcessor.java | 2 +-
  .../hadoop/hive/ql/exec/mr/ExecDriver.java | 2 +-
  .../hadoop/hive/ql/exec/tez/DagUtils.java | 2 +-
  .../hadoop/hive/ql/session/SessionState.java | 6 +-
  .../ql/exec/errors/TestTaskLogProcessor.java | 8 +--
  .../apache/hadoop/hive/ql/hooks/TestHooks.java | 2 +-
  9 files changed, 85 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5098851..27a56dd 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -40,12 +40,11 @@ import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;

  import javax.security.auth.login.LoginException;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
+
+import java.io.*;
  import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashMap;
@@ -96,6 +95,35 @@ public class HiveConf extends Configuration {
      this.isSparkConfigUpdated = isSparkConfigUpdated;
    }

+ public interface EncoderDecoder<K, V> {
+ V encode(K key);
+ K decode(V value);
+ }
+
+ public static class URLEncoderDecoder implements EncoderDecoder<String, String> {
+ private static final String UTF_8 = "UTF-8";
+ @Override
+ public String encode(String key) {
+ try {
+ return URLEncoder.encode(key, UTF_8);
+ } catch (UnsupportedEncodingException e) {
+ return key;
+ }
+ }
+
+ @Override
+ public String decode(String value) {
+ try {
+ return URLDecoder.decode(value, UTF_8);
+ } catch (UnsupportedEncodingException e) {
+ return value;
+ }
+ }
+ }
+ public static class EncoderDecoderFactory {
+ public static final URLEncoderDecoder URL_ENCODER_DECODER = new URLEncoderDecoder();
+ }
+
    static {
      ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
      if (classLoader == null) {
@@ -3283,10 +3311,8 @@ public class HiveConf extends Configuration {

    public static String getVar(Configuration conf, ConfVars var) {
      assert (var.valClass == String.class) : var.varname;
- if (var.altName != null) {
- return conf.get(var.varname, conf.get(var.altName, var.defaultStrVal));
- }
- return conf.get(var.varname, var.defaultStrVal);
+ return var.altName != null ? conf.get(var.varname, conf.get(var.altName, var.defaultStrVal))
+ : conf.get(var.varname, var.defaultStrVal);
    }

    public static String getTrimmedVar(Configuration conf, ConfVars var) {
@@ -3309,10 +3335,13 @@ public class HiveConf extends Configuration {
    }

    public static String getVar(Configuration conf, ConfVars var, String defaultVal) {
- if (var.altName != null) {
- return conf.get(var.varname, conf.get(var.altName, defaultVal));
- }
- return conf.get(var.varname, defaultVal);
+ String ret = var.altName != null ? conf.get(var.varname, conf.get(var.altName, defaultVal))
+ : conf.get(var.varname, defaultVal);
+ return ret;
+ }
+
+ public static String getVar(Configuration conf, ConfVars var, EncoderDecoder<String, String> encoderDecoder) {
+ return encoderDecoder.decode(getVar(conf, var));
    }

    public String getLogIdVar(String defaultValue) {
@@ -3333,6 +3362,10 @@ public class HiveConf extends Configuration {
      assert (var.valClass == String.class) : var.varname;
      conf.set(var.varname, val);
    }
+ public static void setVar(Configuration conf, ConfVars var, String val,
+ EncoderDecoder<String, String> encoderDecoder) {
+ setVar(conf, var, encoderDecoder.encode(val));
+ }

    public static ConfVars getConfVars(String name) {
      return vars.get(name);
@@ -3350,6 +3383,21 @@ public class HiveConf extends Configuration {
      setVar(this, var, val);
    }

+ public String getQueryString() {
+ return getQueryString(this);
+ }
+
+ public static String getQueryString(Configuration conf) {
+ return getVar(conf, ConfVars.HIVEQUERYSTRING, EncoderDecoderFactory.URL_ENCODER_DECODER);
+ }
+
+ public void setQueryString(String query) {
+ setQueryString(this, query);
+ }
+
+ public static void setQueryString(Configuration conf, String query) {
+ setVar(conf, ConfVars.HIVEQUERYSTRING, query, EncoderDecoderFactory.URL_ENCODER_DECODER);
+ }
    public void logVars(PrintStream ps) {
      for (ConfVars one : ConfVars.values()) {
        ps.println(one.varname + "=" + ((get(one.varname) != null) ? get(one.varname) : ""));
@@ -3904,7 +3952,7 @@ public class HiveConf extends Configuration {
      }

      private static boolean isAllowed(Configuration conf, ConfVars setting) {
- String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, null);
+ String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, (String)null);
        return (mode != null) ? !"strict".equals(mode) : !HiveConf.getBoolVar(conf, setting);
      }
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
index 365d500..f88573f 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
@@ -25,6 +25,8 @@ import org.apache.hive.common.util.HiveTestUtils;
  import org.junit.Assert;
  import org.junit.Test;

+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
  import java.util.concurrent.TimeUnit;


@@ -156,4 +158,12 @@ public class TestHiveConf {
      conf.setSparkConfigUpdated(false);
      Assert.assertFalse(conf.getSparkConfigUpdated());
    }
+ @Test
+ public void testEncodingDecoding() throws UnsupportedEncodingException {
+ HiveConf conf = new HiveConf();
+ String query = "select blah, '\u0001' from random_table";
+ conf.setQueryString(query);
+ Assert.assertEquals(URLEncoder.encode(query, "UTF-8"), conf.get(ConfVars.HIVEQUERYSTRING.varname));
+ Assert.assertEquals(query, conf.getQueryString());
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3253146..b50c5a2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -86,7 +86,6 @@ import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter;
  import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
  import org.apache.hadoop.hive.ql.parse.ASTNode;
  import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.CalcitePlanner;
  import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo;
  import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook;
  import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
@@ -499,7 +498,7 @@ public class Driver implements CommandProcessor {
        plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
          SessionState.get().getHiveOperation(), schema);

- conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);
+ conf.setQueryString(queryStr);

        conf.set("mapreduce.workflow.id", "hive_" + queryId);
        conf.set("mapreduce.workflow.name", queryStr);
@@ -1484,7 +1483,7 @@ public class Driver implements CommandProcessor {
      String queryId = plan.getQueryId();
      // Get the query string from the conf file as the compileInternal() method might
      // hide sensitive information during query redaction.
- String queryStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
+ String queryStr = conf.getQueryString();

      maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER);


http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
index 68123d4..b788d8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
@@ -53,7 +53,7 @@ public class TaskLogProcessor {
    private final String query;

    public TaskLogProcessor(JobConf conf) {
- query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
+ query = HiveConf.getQueryString(conf);

      heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats());
      heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats());

http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index b184b4e..ce020a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -568,7 +568,7 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
      // Intentionally overwrites anything the user may have put here
      conf.setBoolean("hive.input.format.sorted", mWork.isInputFormatSorted());

- if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, null) == null) {
+ if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, (String)null) == null) {
        HiveConf.setVar(conf, ConfVars.HIVE_CURRENT_DATABASE, getCurrentDB());
      }
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 473dbd6..79da860 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -904,7 +904,7 @@ public class DagUtils {

    public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException {
      FileStatus fstatus = null;
- String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, null);
+ String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, (String)null);
      if (hdfsDirPathStr != null) {
        LOG.info("Hive jar directory is " + hdfsDirPathStr);
        fstatus = validateTargetDir(new Path(hdfsDirPathStr), conf);

http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 109cd8c..78bbb1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -373,11 +373,11 @@ public class SessionState {
    }

    public void setCmd(String cmdString) {
- conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, cmdString);
+ conf.setQueryString(cmdString);
    }

    public String getCmd() {
- return (conf.getVar(HiveConf.ConfVars.HIVEQUERYSTRING));
+ return (conf.getQueryString());
    }

    public String getQueryId() {
@@ -1642,7 +1642,7 @@ public class SessionState {
      // Provide a facility to set current timestamp during tests
      if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) {
        String overrideTimestampString =
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, null);
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, (String)null);
        if (overrideTimestampString != null && overrideTimestampString.length() > 0) {
          queryCurrentTimestamp = Timestamp.valueOf(overrideTimestampString);
        }

http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
index 67a86a6..477479d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java
@@ -98,7 +98,7 @@ public class TestTaskLogProcessor {
    @Test
    public void testGetStackTraces() throws Exception {
      JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");

      final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);

@@ -150,7 +150,7 @@ public class TestTaskLogProcessor {
    @Test
    public void testScriptErrorHeuristic() throws Exception {
      JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");

      final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);

@@ -177,7 +177,7 @@ public class TestTaskLogProcessor {
    @Test
    public void testDataCorruptErrorHeuristic() throws Exception {
      JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");

      final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);

@@ -210,7 +210,7 @@ public class TestTaskLogProcessor {
    @Test
    public void testMapAggrMemErrorHeuristic() throws Exception {
      JobConf jobConf = new JobConf();
- jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;");
+ HiveConf.setQueryString(jobConf, "select * from foo group by moo;");

      final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf);


http://git-wip-us.apache.org/repos/asf/hive/blob/415373bb/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
index 8d27762..4c14d8b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java
@@ -73,7 +73,7 @@ public class TestHooks {
      Driver driver = createDriver(conf);
      int ret = driver.compile("select 'XXX' from t1");
      assertEquals("Checking command success", 0, ret);
- assertEquals("select 'AAA' from t1", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING));
+ assertEquals("select 'AAA' from t1", conf.getQueryString());
    }

    public static class SimpleQueryRedactor extends Redactor {

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 9, '16 at 8:47a
activeMar 9, '16 at 8:47a
posts1
users1
websitehive.apache.org

1 user in discussion

Amareshwari: 1 post

People

Translate

site design / logo © 2021 Grokbase