FAQ
Repository: hive
Updated Branches:
   refs/heads/llap 134f2f749 -> 28d1082b4


HIVE-13389: LLAP external submission client ends up attempting to find an LLAP instance based on the submitting user instead of the hive user


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/28d1082b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/28d1082b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/28d1082b

Branch: refs/heads/llap
Commit: 28d1082b4a2acfa46beddc52e4d3944e783b6f21
Parents: 134f2f7
Author: Jason Dere <jdere@hortonworks.com>
Authored: Wed Mar 30 14:35:23 2016 -0700
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Wed Mar 30 14:35:23 2016 -0700

----------------------------------------------------------------------
  .../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
  .../org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java | 4 +++-
  .../llap/registry/impl/LlapZookeeperRegistryImpl.java | 9 ++++++++-
  .../org/apache/hadoop/hive/llap/LlapInputFormat.java | 3 +++
  ql/pom.xml | 6 ++++++
  .../java/org/apache/hadoop/hive/llap/LlapInputSplit.java | 11 +++++++++--
  .../hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java | 4 +++-
  7 files changed, 35 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 88b4387..ef32b2b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2591,6 +2591,9 @@ public class HiveConf extends Configuration {
          "ZooKeeper for ZooKeeper SecretManager."),
      LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "",
          "ZooKeeper connection string for ZooKeeper SecretManager."),
+ LLAP_ZK_REGISTRY_USER("hive.llap.zk.registry.user", "",
+ "In the LLAP ZooKeeper-based registry, specifies the username in the Zookeeper path.\n" +
+ "This should be the hive user or whichever user is running the LLAP daemon."),
      // Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
      // resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
      LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."),

http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
index b8b69a7..338930e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
@@ -46,7 +46,8 @@ public class TestLlapInputSplit {
          planBytes,
          fragmentBytes,
          locations,
- schema);
+ schema,
+ "hive");
      ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
      DataOutputStream dataOut = new DataOutputStream(byteOutStream);
      split1.write(dataOut);
@@ -94,6 +95,7 @@ public class TestLlapInputSplit {
      }
      assertArrayEquals(split1.getLocations(), split2.getLocations());
      assertEquals(split1.getSchema(), split2.getSchema());
+ assertEquals(split1.getLlapUser(), split2.getLlapUser());
    }

  }

http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index c440e1e..0536589 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@ -176,7 +176,7 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
      // worker does not respond due to communication interruptions it will retain the same sequence
      // number when it returns back. If session timeout expires, the node will be deleted and new
      // addition of the same node (restart) will get next sequence number
- this.pathPrefix = "/" + RegistryUtils.currentUser() + "/" + instanceName + "/workers/worker-";
+ this.pathPrefix = "/" + getZkPathUser(this.conf) + "/" + instanceName + "/workers/worker-";
      this.instancesCache = null;
      this.instances = null;
      this.stateChangeListeners = new HashSet<>();
@@ -210,6 +210,13 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
      return quorum.toString();
    }

+ private String getZkPathUser(Configuration conf) {
+ // External LLAP clients would need to set LLAP_ZK_REGISTRY_USER to the LLAP daemon user (hive),
+ // rather than relying on RegistryUtils.currentUser().
+ String user = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, RegistryUtils.currentUser());
+ return user;
+ }
+
    public Endpoint getRpcEndpoint() {
      final int rpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_RPC_PORT);
      return RegistryTypeUtils.ipcEndpoint(IPC_LLAP, new InetSocketAddress(hostname, rpcPort));

http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
index 7f11e1d..aaca7d6 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
@@ -88,6 +88,9 @@ public class LlapInputFormat<V extends WritableComparable> implements InputForma
        Reporter reporter) throws IOException {

      LlapInputSplit llapSplit = (LlapInputSplit) split;
+
+ // Set conf to use LLAP user rather than current user for LLAP Zk registry.
+ HiveConf.setVar(job, HiveConf.ConfVars.LLAP_ZK_REGISTRY_USER, llapSplit.getLlapUser());
      SubmitWorkInfo submitWorkInfo = SubmitWorkInfo.fromBytes(llapSplit.getPlanBytes());

      ServiceInstance serviceInstance = getServiceInstance(job, llapSplit);

http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index ebb9599..e0de1fb 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -286,6 +286,12 @@
        <version>${hadoop.version}</version>
        <optional>true</optional>
      </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ </dependency>

      <dependency>
        <groupId>org.apache.ivy</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
index 2ac0ccd..17a0d2d 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
@@ -33,16 +33,18 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
    byte[] fragmentBytes;
    SplitLocationInfo[] locations;
    Schema schema;
+ String llapUser;

    public LlapInputSplit() {
    }

- public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, SplitLocationInfo[] locations, Schema schema) {
+ public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, SplitLocationInfo[] locations, Schema schema, String llapUser) {
      this.planBytes = planBytes;
      this.fragmentBytes = fragmentBytes;
      this.locations = locations;
      this.schema = schema;
      this.splitNum = splitNum;
+ this.llapUser = llapUser;
    }

    public Schema getSchema() {
@@ -102,7 +104,7 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
        throw new IOException(e);
      }

-
+ out.writeUTF(llapUser);
    }

    @Override
@@ -134,10 +136,15 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
      } catch (Exception e) {
        throw new IOException(e);
      }
+ llapUser = in.readUTF();
    }

    @Override
    public SplitLocationInfo[] getLocationInfo() throws IOException {
      return locations;
    }
+
+ public String getLlapUser() {
+ return llapUser;
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index ebb0ca5..6267324 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.io.DataOutputBuffer;
  import org.apache.hadoop.mapred.InputSplit;
  import org.apache.hadoop.mapred.JobConf;
  import org.apache.hadoop.mapred.SplitLocationInfo;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
  import org.apache.hadoop.yarn.api.records.ApplicationId;
  import org.apache.hadoop.yarn.api.records.LocalResource;
  import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -319,6 +320,7 @@ public class GenericUDTFGetSplits extends GenericUDTF {
        ApplicationId fakeApplicationId
          = ApplicationId.newInstance(Math.abs(new Random().nextInt()), 0);

+ String llapUser = RegistryUtils.currentUser();
        LOG.info("Number of splits: " + (eventList.size() - 1));
        for (int i = 0; i < eventList.size() - 1; i++) {

@@ -355,7 +357,7 @@ public class GenericUDTFGetSplits extends GenericUDTF {

          byte[] submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo);

- result[i] = new LlapInputSplit(i, submitWorkBytes, dob.getData(), locations, schema);
+ result[i] = new LlapInputSplit(i, submitWorkBytes, dob.getData(), locations, schema, llapUser);
        }
        return result;
      } catch (Exception e) {

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 30, '16 at 9:35p
activeMar 30, '16 at 9:35p
posts1
users1
websitehive.apache.org

1 user in discussion

Jdere: 1 post

People

Translate

site design / logo © 2021 Grokbase