FAQ
HIVE-12270: Add DBTokenStore support to HS2 delegation token (Chaoyu Tang, reviewed by Szehon Ho)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/87131d0c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/87131d0c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/87131d0c

Branch: refs/heads/master
Commit: 87131d0c7cce973d8792ed354f70ec73b0f52dcd
Parents: 41e8201
Author: ctang <ctang@cloudera.com>
Authored: Tue Mar 8 15:05:54 2016 -0500
Committer: ctang <ctang@cloudera.com>
Committed: Tue Mar 8 15:05:54 2016 -0500

----------------------------------------------------------------------
  .../org/apache/hive/minikdc/MiniHiveKdc.java | 15 +
  .../hive/minikdc/TestJdbcWithDBTokenStore.java | 40 +
  .../hive/minikdc/TestJdbcWithMiniKdc.java | 12 +-
  .../hadoop/hive/thrift/TestDBTokenStore.java | 3 +-
  metastore/if/hive_metastore.thrift | 24 +
  .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 22340 ++++++++++-------
  .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 1409 +-
  .../ThriftHiveMetastore_server.skeleton.cpp | 40 +
  .../hive/metastore/api/ThriftHiveMetastore.java | 9220 ++++++-
  .../gen-php/metastore/ThriftHiveMetastore.php | 1756 ++
  .../hive_metastore/ThriftHiveMetastore-remote | 56 +
  .../hive_metastore/ThriftHiveMetastore.py | 1639 +-
  .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 450 +
  .../hadoop/hive/metastore/HiveMetaStore.java | 159 +
  .../hive/metastore/HiveMetaStoreClient.java | 42 +
  .../hadoop/hive/metastore/IMetaStoreClient.java | 19 +-
  .../gen-py/TCLIService/TCLIService-remote | 0
  .../hive/service/auth/HiveAuthFactory.java | 33 +-
  .../apache/hadoop/hive/thrift/DBTokenStore.java | 49 +-
  .../hive/thrift/HiveDelegationTokenManager.java | 1 +
  20 files changed, 26383 insertions(+), 10924 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
----------------------------------------------------------------------
diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
index dedbf35..4e3a9c5 100644
--- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
+++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
@@ -179,5 +179,20 @@ public class MiniHiveKdc {
          withMiniKdc(hivePrincipal, hiveKeytab).build();
    }

+ /**
+ * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc
+ * @param miniHiveKdc
+ * @param hiveConf
+ * @return new MiniHS2 instance
+ * @throws Exception
+ */
+ public static MiniHS2 getMiniHS2WithKerbWithRemoteHMS(MiniHiveKdc miniHiveKdc, HiveConf hiveConf) throws Exception {
+ String hivePrincipal =
+ miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL);
+ String hiveKeytab = miniHiveKdc.getKeyTabFile(
+ miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL));

+ return new MiniHS2.Builder().withConf(hiveConf).withRemoteMetastore().
+ withMiniKdc(hivePrincipal, hiveKeytab).build();
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java
new file mode 100644
index 0000000..d690aaa
--- /dev/null
+++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.minikdc;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.BeforeClass;
+
+public class TestJdbcWithDBTokenStore extends TestJdbcWithMiniKdc{
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ Class.forName(MiniHS2.getJdbcDriverName());
+ confOverlay.put(ConfVars.HIVE_SERVER2_SESSION_HOOK.varname,
+ SessionHookTest.class.getName());
+
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setVar(ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS, "org.apache.hadoop.hive.thrift.DBTokenStore");
+ miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf);
+ miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf);
+ miniHS2.start(confOverlay);
+ }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
----------------------------------------------------------------------
diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
index 3ef2ce3..71a08fb 100644
--- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
+++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
@@ -57,10 +57,10 @@ public class TestJdbcWithMiniKdc {
      }
    }

- private static MiniHS2 miniHS2 = null;
- private static MiniHiveKdc miniHiveKdc = null;
- private static Map<String, String> confOverlay = new HashMap<String, String>();
- private Connection hs2Conn;
+ protected static MiniHS2 miniHS2 = null;
+ protected static MiniHiveKdc miniHiveKdc = null;
+ protected static Map<String, String> confOverlay = new HashMap<String, String>();
+ protected Connection hs2Conn;

    @BeforeClass
    public static void beforeTest() throws Exception {
@@ -241,7 +241,7 @@ public class TestJdbcWithMiniKdc {
     * @param expectedValue
     * @throws Exception
     */
- private void verifyProperty(String propertyName, String expectedValue) throws Exception {
+ protected void verifyProperty(String propertyName, String expectedValue) throws Exception {
      Statement stmt = hs2Conn .createStatement();
      ResultSet res = stmt.executeQuery("set " + propertyName);
      assertTrue(res.next());
@@ -251,7 +251,7 @@ public class TestJdbcWithMiniKdc {
    }

    // Store the given token in the UGI
- private void storeToken(String tokenStr, UserGroupInformation ugi)
+ protected void storeToken(String tokenStr, UserGroupInformation ugi)
        throws Exception {
      Utils.setTokenStr(ugi,
          tokenStr, HiveAuthFactory.HS2_CLIENT_TOKEN);

http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
index f5934ee..4bfa224 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
  import org.apache.hadoop.hive.metastore.api.MetaException;
  import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
  import org.apache.hadoop.hive.thrift.DelegationTokenStore.TokenStoreException;
+import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode;
  import org.apache.hadoop.io.Text;
  import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
  import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport;
@@ -37,7 +38,7 @@ public class TestDBTokenStore extends TestCase{
    public void testDBTokenStore() throws TokenStoreException, MetaException, IOException {

      DelegationTokenStore ts = new DBTokenStore();
- ts.init(new HMSHandler("Test handler"), null);
+ ts.init(new HMSHandler("Test handler"), ServerMode.METASTORE);
      assertEquals(0, ts.getMasterKeys().length);
      assertEquals(false,ts.removeMasterKey(-1));
      try{

http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
index 3635054..6a55962 100755
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@ -1294,6 +1294,30 @@ service ThriftHiveMetastore extends fb303.FacebookService
    // method to cancel delegation token obtained from metastore server
    void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)

+ // add a delegation token
+ bool add_token(1:string token_identifier, 2:string delegation_token)
+
+ // remove a delegation token
+ bool remove_token(1:string token_identifier)
+
+ // get a delegation token by identifier
+ string get_token(1:string token_identifier)
+
+ // get all delegation token identifiers
+ list<string> get_all_token_identifiers()
+
+ // add master key
+ i32 add_master_key(1:string key) throws (1:MetaException o1)
+
+ // update master key
+ void update_master_key(1:i32 seq_number, 2:string key) throws (1:NoSuchObjectException o1, 2:MetaException o2)
+
+ // remove master key
+ bool remove_master_key(1:i32 key_seq)
+
+ // get master keys
+ list<string> get_master_keys()
+
    // Transaction and lock management calls
    // Get just list of open transactions
    GetOpenTxnsResponse get_open_txns()

Search Discussions

Discussion Posts

Previous

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 5 of 7 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 8, '16 at 8:06p
activeMar 8, '16 at 8:06p
posts7
users1
websitehive.apache.org

1 user in discussion

Ctang: 7 posts

People

Translate

site design / logo © 2021 Grokbase