FAQ
Repository: hive
Updated Branches:
   refs/heads/master baf32e2e2 -> 01580af2e


http://git-wip-us.apache.org/repos/asf/hive/blob/01580af2/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
index f4f30d7..f4df2e2 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.metastore.hbase;
  import com.google.common.collect.Lists;
  import com.google.protobuf.ByteString;
  import com.google.protobuf.InvalidProtocolBufferException;
-
  import org.apache.commons.lang.StringUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@ -74,10 +73,8 @@ import java.io.IOException;
  import java.nio.charset.Charset;
  import java.nio.charset.StandardCharsets;
  import java.security.MessageDigest;
-import java.util.ArrayDeque;
  import java.util.ArrayList;
  import java.util.Arrays;
-import java.util.Deque;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
@@ -847,15 +844,67 @@ class HBaseUtils {
    }

    /**
+ * Deserialize a partition key when you know nothing about it. That is, you do not know what
+ * dbname, tablename it came from.
+ * @param key the key fetched from HBase
+ * @param callback A reference to the calling HBaseReadWrite object. This has to be done as a
+ * callback because we have to first deserialize the database name and table
+ * name, and then fetch the table information, and then we will know how to
+ * desierliaze the rest of the key.
+ * @return a list that includes the dbname, tablename, and partition values
+ * @throws IOException
+ */
+ static List<String> deserializePartitionKey(byte[] key, HBaseReadWrite callback)
+ throws IOException {
+ List<String> keyParts =
+ desierliazeDbNameTableNameFromPartitionKey(key, callback.getConf());
+ Table table = callback.getTable(keyParts.get(0), keyParts.get(1));
+ keyParts.addAll(deserializePartitionKey(table.getPartitionKeys(), key, callback.getConf()));
+ return keyParts;
+ }
+
+ /**
     * Deserialize a partition. This version should be used when the partition key is not already
- * known (eg a scan).
+ * known and the database and table name are not known either (eg a full scan). Because the
+ * dbname and tablename (and thus the partition columns) are not known a priori this version
+ * has to go fetch the table after it figures out which table. If you already have the table
+ * object you should use
+ * {@link #deserializePartition(String,String,List,byte[],byte[],Configuration)}
     * @param key the key fetched from HBase
     * @param serialized the value fetched from HBase
+ * @param callback A reference to the calling HBaseReadWrite object. This has to be done as a
+ * callback because we have to first deserialize the database name and table
+ * name, and then fetch the table information, and then we will know how to
+ * desierliaze the rest of the key.
     * @return A struct that contains the partition plus parts of the storage descriptor
     */
- static StorageDescriptorParts deserializePartition(String dbName, String tableName, List<FieldSchema> partitions,
- byte[] key, byte[] serialized, Configuration conf) throws InvalidProtocolBufferException {
- List keys = deserializePartitionKey(partitions, key, conf);
+ static StorageDescriptorParts deserializePartition(byte[] key, byte[] serialized,
+ HBaseReadWrite callback)
+ throws IOException {
+ List<String> dbNameTableName =
+ desierliazeDbNameTableNameFromPartitionKey(key, callback.getConf());
+ Table table = callback.getTable(dbNameTableName.get(0), dbNameTableName.get(1));
+ List<String> keys = deserializePartitionKey(table.getPartitionKeys(), key, callback.getConf());
+ return deserializePartition(dbNameTableName.get(0), dbNameTableName.get(1), keys, serialized);
+ }
+
+ /**
+ * Deserialize a partition. This version should be used when you know the dbname and tablename
+ * but not the partition values.
+ * @param dbName database this partition is in
+ * @param tableName table this partition is in
+ * @param partitions schemas for the partition columns of this table
+ * @param key key fetched from HBase
+ * @param serialized serialized version of the partition
+ * @param conf configuration file
+ * @return
+ * @throws InvalidProtocolBufferException
+ */
+ static StorageDescriptorParts deserializePartition(String dbName, String tableName,
+ List<FieldSchema> partitions, byte[] key,
+ byte[] serialized, Configuration conf)
+ throws InvalidProtocolBufferException {
+ List<String> keys = deserializePartitionKey(partitions, key, conf);
      return deserializePartition(dbName, tableName, keys, serialized);
    }

@@ -887,12 +936,46 @@ class HBaseUtils {
      return sdParts;
    }

- private static String[] deserializeKey(byte[] key) {
+ static String[] deserializeKey(byte[] key) {
      String k = new String(key, ENCODING);
      return k.split(KEY_SEPARATOR_STR);
    }

- static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key,
+ private static List<String> desierliazeDbNameTableNameFromPartitionKey(byte[] key,
+ Configuration conf) {
+ StringBuffer names = new StringBuffer();
+ names.append("dbName,tableName,");
+ StringBuffer types = new StringBuffer();
+ types.append("string,string,");
+ BinarySortableSerDe serDe = new BinarySortableSerDe();
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
+ try {
+ serDe.initialize(conf, props);
+ List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(0, 2);
+ List<String> keys = new ArrayList<>();
+ for (int i=0;i<deserializedkeys.size();i++) {
+ Object deserializedKey = deserializedkeys.get(i);
+ if (deserializedKey==null) {
+ throw new RuntimeException("Can't have a null dbname or tablename");
+ } else {
+ TypeInfo inputType = TypeInfoUtils.getTypeInfoFromTypeString("string");
+ ObjectInspector inputOI =
+ TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
+ Converter converter = ObjectInspectorConverters.getConverter(inputOI,
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ keys.add((String) converter.convert(deserializedKey));
+ }
+ }
+ return keys;
+ } catch (SerDeException e) {
+ throw new RuntimeException("Error when deserialize key", e);
+ }
+ }
+
+ // Deserialize a partition key and return _only_ the partition values.
+ private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key,
        Configuration conf) {
      StringBuffer names = new StringBuffer();
      names.append("dbName,tableName,");
@@ -911,7 +994,7 @@ class HBaseUtils {
      props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
      props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
      try {
- serDe.initialize(new Configuration(), props);
+ serDe.initialize(conf, props);
        List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size()+2);
        List<String> partitionKeys = new ArrayList<String>();
        for (int i=0;i<deserializedkeys.size();i++) {

Search Discussions

  • Gates at Oct 28, 2015 at 7:39 pm
    HIVE-11564 HBaseSchemaTool should be able to list objects (gates reviewed by Daniel Dai)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/01580af2
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/01580af2
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/01580af2

    Branch: refs/heads/master
    Commit: 01580af2e2401d0512ec95d52c55aeb61c116039
    Parents: baf32e2
    Author: Alan Gates <gates@hortonworks.com>
    Authored: Wed Oct 28 12:38:16 2015 -0700
    Committer: Alan Gates <gates@hortonworks.com>
    Committed: Wed Oct 28 12:38:16 2015 -0700

    ----------------------------------------------------------------------
      .../metastore/hbase/TestHBaseSchemaTool.java | 584 ++++++++++++++++
      .../metastore/hbase/TestHBaseSchemaTool2.java | 61 ++
      .../hive/metastore/hbase/HBaseReadWrite.java | 698 +++++++++++++++----
      .../hive/metastore/hbase/HBaseSchemaTool.java | 282 ++++----
      .../hadoop/hive/metastore/hbase/HBaseUtils.java | 103 ++-
      5 files changed, 1431 insertions(+), 297 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/01580af2/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
    new file mode 100644
    index 0000000..d3b8615
    --- /dev/null
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
    @@ -0,0 +1,584 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing,
    + * software distributed under the License is distributed on an
    + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    + * KIND, either express or implied. See the License for the
    + * specific language governing permissions and limitations
    + * under the License.
    + */
    +package org.apache.hadoop.hive.metastore.hbase;
    +
    +import org.apache.commons.lang.StringUtils;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
    +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
    +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
    +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
    +import org.apache.hadoop.hive.metastore.api.Database;
    +import org.apache.hadoop.hive.metastore.api.FieldSchema;
    +import org.apache.hadoop.hive.metastore.api.Function;
    +import org.apache.hadoop.hive.metastore.api.FunctionType;
    +import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
    +import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
    +import org.apache.hadoop.hive.metastore.api.HiveObjectType;
    +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
    +import org.apache.hadoop.hive.metastore.api.Partition;
    +import org.apache.hadoop.hive.metastore.api.PrincipalType;
    +import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
    +import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
    +import org.apache.hadoop.hive.metastore.api.Role;
    +import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
    +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
    +import org.apache.hadoop.hive.metastore.api.Table;
    +import org.junit.AfterClass;
    +import org.junit.Assert;
    +import org.junit.Before;
    +import org.junit.BeforeClass;
    +import org.junit.Test;
    +
    +import java.io.ByteArrayOutputStream;
    +import java.io.IOException;
    +import java.io.PrintStream;
    +import java.util.ArrayList;
    +import java.util.Arrays;
    +import java.util.Collections;
    +import java.util.List;
    +
    +public class TestHBaseSchemaTool extends HBaseIntegrationTests {
    + private static final Log LOG = LogFactory.getLog(TestHBaseSchemaTool.class.getName());
    + private String lsep = System.getProperty("line.separator");
    +
    + @BeforeClass
    + public static void startup() throws Exception {
    + HBaseIntegrationTests.startMiniCluster();
    + }
    +
    + @AfterClass
    + public static void shutdown() throws Exception {
    + HBaseIntegrationTests.shutdownMiniCluster();
    + }
    +
    + @Before
    + public void setup() throws IOException {
    + setupHBaseStore();
    + }
    +
    + @Test
    + public void listTables() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(true, null, null, null, conf, out, err);
    + Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep,
    + outStr.toString());
    + }
    +
    + @Test
    + public void bogusTable() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, "nosuch", null, null, conf, out, err);
    + Assert.assertEquals("Unknown table: nosuch" + lsep, errStr.toString());
    + }
    +
    + @Test
    + public void noSuchDb() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such database: nosuch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noMatchingDb() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, null, "nomatch", conf, out, err);
    + Assert.assertEquals("No matching database: nomatch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchRole() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such role: nosuch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noMatchingRole() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, null, "nomatch", conf, out, err);
    + Assert.assertEquals("No matching role: nomatch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchUser() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such user: nosuch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noMatchingUser() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, "nomatch", conf, out, err);
    + Assert.assertEquals("No matching user: nomatch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchFunction() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such function: nosuch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noMatchingFunction() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, null, "nomatch", conf, out,
    + err);
    + Assert.assertEquals("No matching function: nomatch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchTable() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such table: nosuch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noMatchingTable() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, null, "nomatch", conf, out, err);
    + Assert.assertEquals("No matching table: nomatch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchPart() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such partition: nosuch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchPartValidFormat() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    + // Test with something that looks like a valid entry
    + new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "default.nosuch.nosuch", null, conf,
    + out, err);
    + Assert.assertEquals("No such partition: default.nosuch.nosuch" + lsep, outStr.toString());
    + }
    +
    +
    + @Test
    + public void noMatchingPart() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch", conf, out, err);
    + Assert.assertEquals("No matching partition: nomatch" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noMatchingPartValidFormat() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch.a.b", conf, out, err);
    + Assert.assertEquals("No matching partition: nomatch.a.b" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void noSuchStorageDescriptor() throws Exception {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + // Strangly enough things don't come back quite the same when going through the Base64
    + // encode/decode.
    + new HBaseSchemaTool().go(false, HBaseReadWrite.SD_TABLE, "nosuch", null, conf, out, err);
    + Assert.assertEquals("No such storage descriptor: nosucg" + lsep, outStr.toString());
    + }
    +
    + @Test
    + public void oneMondoTest() throws Exception {
    + // This is a pain to do in one big test, but we have to control the order so that we have tests
    + // without dbs, etc.
    + HBaseSchemaTool tool = new HBaseSchemaTool();
    +
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + // This needs to be up front before we create any tables or partitions
    + tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err);
    + Assert.assertEquals("No storage descriptors" + lsep, outStr.toString());
    +
    + // This one needs to be up front too
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err);
    + Assert.assertEquals("No sequences" + lsep, outStr.toString());
    +
    + // Create some databases
    + String[] dbNames = new String[3];
    + for (int i = 0; i < dbNames.length; i++) {
    + dbNames[i] = "db" + i;
    + Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
    + store.createDatabase(db);
    + }
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.DB_TABLE, "db0", null, conf, out, err);
    + Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," +
    + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.DB_TABLE, null, ".*", conf, out, err);
    + Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," +
    + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
    + "{\"name\":\"db1\",\"description\":\"no description\"," +
    + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
    + "{\"name\":\"db2\",\"description\":\"no description\"," +
    + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep,
    + outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.DB_TABLE, null, "db[12]", conf, out, err);
    + Assert.assertEquals("{\"name\":\"db1\",\"description\":\"no description\"," +
    + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
    + "{\"name\":\"db2\",\"description\":\"no description\"," +
    + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep,
    + outStr.toString());
    +
    + String[] roleNames = new String[2];
    + for (int i = 0; i < roleNames.length; i++) {
    + roleNames[i] = "role" + i;
    + store.addRole(roleNames[i], "me");
    + }
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.ROLE_TABLE, null, "role.", conf, out, err);
    + Assert.assertEquals("{\"roleName\":\"role0\",\"createTime\":now,\"ownerName\":\"me\"}" +
    + lsep + "{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep,
    + outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now"));
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.ROLE_TABLE, "role1", null, conf, out, err);
    + Assert.assertEquals("{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep,
    + outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now"));
    +
    + Role role1 = store.getRole("role1");
    + store.grantRole(role1, "fred", PrincipalType.USER, "me", PrincipalType.USER, false);
    + store.grantRole(role1, "joanne", PrincipalType.USER, "me", PrincipalType.USER, false);
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, ".*", conf, out, err);
    + Assert.assertEquals("fred: role1" + lsep + "joanne: role1" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "joanne", null, conf, out, err);
    + Assert.assertEquals("role1" + lsep, outStr.toString());
    +
    + String[] funcNames = new String[3];
    + for (int i = 0; i < funcNames.length; i++) {
    + funcNames[i] = "func" + i;
    + Function function = new Function(funcNames[i], "db1", "Function", "me", PrincipalType.USER, 0,
    + FunctionType.JAVA, null);
    + store.createFunction(function);
    + }
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.FUNC_TABLE, "db1.func0", null, conf, out, err);
    + Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," +
    + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
    + "\"functionType\":1}" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.FUNC_TABLE, null, ".*", conf, out, err);
    + Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," +
    + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
    + "\"functionType\":1}" + lsep +
    + "{\"functionName\":\"func1\",\"dbName\":\"db1\"," +
    + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
    + "\"functionType\":1}" + lsep +
    + "{\"functionName\":\"func2\",\"dbName\":\"db1\"," +
    + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
    + "\"functionType\":1}" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.FUNC_TABLE, null, "db1.func[12]", conf, out, err);
    + Assert.assertEquals("{\"functionName\":\"func1\",\"dbName\":\"db1\"," +
    + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
    + "\"functionType\":1}" + lsep +
    + "{\"functionName\":\"func2\",\"dbName\":\"db1\"," +
    + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
    + "\"functionType\":1}" + lsep, outStr.toString());
    +
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err);
    + Assert.assertEquals("No global privileges" + lsep, outStr.toString());
    +
    + List<HiveObjectPrivilege> privileges = new ArrayList<>();
    + HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.GLOBAL, "db0", "tab0", null,
    + null);
    + PrivilegeGrantInfo grantInfo =
    + new PrivilegeGrantInfo("read", 0, "me", PrincipalType.USER, false);
    + HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER,
    + grantInfo);
    + privileges.add(hop);
    +
    + grantInfo = new PrivilegeGrantInfo("create", 0, "me", PrincipalType.USER, true);
    + hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER, grantInfo);
    + privileges.add(hop);
    +
    + PrivilegeBag pBag = new PrivilegeBag(privileges);
    + store.grantPrivileges(pBag);
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err);
    + Assert.assertEquals(
    + "{\"userPrivileges\":{\"user\":[{\"privilege\":\"read\",\"createTime\":0," +
    + "\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":0},{\"privilege\":\"create\"," +
    + "\"createTime\":0,\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":1}]}}" + lsep,
    + outStr.toString());
    +
    +
    + String[] tableNames = new String[3];
    + for (int i = 0; i < tableNames.length; i++) {
    + tableNames[i] = "tab" + i;
    + StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int",
    + ""), new FieldSchema("col2", "varchar(32)", "")),
    + "/tmp", null, null, false, 0, null, null, null, Collections.<String, String>emptyMap());
    + Table tab = new Table(tableNames[i], dbNames[0], "me", 0, 0, 0, sd,
    + Arrays.asList(new FieldSchema("pcol1", "string", ""),
    + new FieldSchema("pcol2", "string", "")),
    + Collections.<String, String>emptyMap(), null, null, null);
    + store.createTable(tab);
    + }
    +
    + ColumnStatisticsDesc tableStatsDesc = new ColumnStatisticsDesc(false, "db0", "tab0");
    + ColumnStatisticsData tcsd = new ColumnStatisticsData();
    + LongColumnStatsData tlcsd = new LongColumnStatsData(1, 2);
    + tlcsd.setLowValue(-95);
    + tlcsd.setHighValue(95);
    + tcsd.setLongStats(tlcsd);
    + ColumnStatisticsData tcsd2 = new ColumnStatisticsData();
    + tcsd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397));
    + List<ColumnStatisticsObj> tcsos = Arrays.asList(
    + new ColumnStatisticsObj("col1", "int", tcsd),
    + new ColumnStatisticsObj("col2", "varchar(32)", tcsd2));
    + ColumnStatistics tStatObj = new ColumnStatistics(tableStatsDesc, tcsos);
    + store.updateTableColumnStatistics(tStatObj);
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.TABLE_TABLE, "db0.tab1", null, conf, out, err);
    + Assert.assertEquals("{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\"," +
    + "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," +
    + "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," +
    + "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{}," +
    + "\"tableType\":\"\"} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep,
    + outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.TABLE_TABLE, null, "db0.*", conf, out, err);
    + Assert.assertEquals("{\"tableName\":\"tab0\",\"dbName\":\"db0\",\"owner\":\"me\"," +
    + "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," +
    + "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," +
    + "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{}," +
    + "\"tableType\":\"\"} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " +
    + "col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
    + "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
    + "\"numDVs\":2}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
    + "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78," +
    + "\"numNulls\":29,\"numDVs\":397}}}" + lsep +
    + "{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," +
    + "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," +
    + "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," +
    + "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\"} sdHash: " +
    + "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep +
    + "{\"tableName\":\"tab2\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," +
    + "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," +
    + "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," +
    + "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\"} sdHash: " +
    + "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString());
    +
    + List<List<String>> partVals = Arrays.asList(Arrays.asList("a", "b"), Arrays.asList("c", "d"));
    + for (List<String> pv : partVals) {
    + StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int",
    + ""), new FieldSchema("col2", "varchar(32)", "")),
    + "/tmp", null, null, false, 0, null, null, null, Collections.<String, String>emptyMap());
    + Partition p = new Partition(pv, "db0", "tab1", 0, 0, sd, Collections.<String, String>emptyMap());
    + store.addPartition(p);
    + }
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.PART_TABLE, "db0.tab1.a.b", null, conf, out, err);
    + Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
    + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: " +
    + "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString());
    +
    + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db0", "tab1");
    + statsDesc.setPartName("pcol1=c/pcol2=d");
    + ColumnStatisticsData csd1 = new ColumnStatisticsData();
    + LongColumnStatsData lcsd = new LongColumnStatsData(1, 2);
    + lcsd.setLowValue(-95);
    + lcsd.setHighValue(95);
    + csd1.setLongStats(lcsd);
    + ColumnStatisticsData csd2 = new ColumnStatisticsData();
    + csd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397));
    + List<ColumnStatisticsObj> csos = Arrays.asList(
    + new ColumnStatisticsObj("col1", "int", csd1),
    + new ColumnStatisticsObj("col2", "varchar(32)", csd2));
    + ColumnStatistics statsObj = new ColumnStatistics(statsDesc, csos);
    + store.updatePartitionColumnStatistics(statsObj, partVals.get(1));
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.PART_TABLE, "db0.tab1.c.d", null, conf, out, err);
    + Assert.assertEquals("{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
    + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
    + "stats: column col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
    + "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
    + "\"numDVs\":2}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
    + "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," +
    + "\"numDVs\":397}}}" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.PART_TABLE, null, "db0.tab1", conf, out, err);
    + Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
    + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
    + "stats:" + lsep +
    + "{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\",\"createTime\":0," +
    + "\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " +
    + "col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
    + "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
    + "\"numDVs\":2}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
    + "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," +
    + "\"numDVs\":397}}}" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.PART_TABLE, null, "db0.tab1.a", conf, out, err);
    + Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
    + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
    + "stats:" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.SD_TABLE, "qQTgZAi5VzgpozzFGmIVTQ", null, conf, out, err);
    + Assert.assertEquals("{\"cols\":[{\"name\":\"col1\",\"type\":\"int\",\"comment\":\"\"}," +
    + "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," +
    + "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep,
    + outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err);
    + Assert.assertEquals("qQTgZAi5VzgpozzFGmIVTQ: {\"cols\":[{\"name\":\"col1\",\"type\":\"int\"," +
    + "\"comment\":\"\"}," +
    + "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," +
    + "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep,
    + outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err);
    + Assert.assertEquals("No security related entries" + lsep, outStr.toString());
    +
    + store.addMasterKey("this be a key");
    + store.addToken("tokenid", "delegation token");
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err);
    + Assert.assertEquals("Master key 0: this be a key" + lsep +
    + "Delegation token tokenid: delegation token" + lsep, outStr.toString());
    +
    + outStr = new ByteArrayOutputStream();
    + out = new PrintStream(outStr);
    + tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err);
    + Assert.assertEquals("mk: 1" + lsep, outStr.toString());
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/01580af2/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
    new file mode 100644
    index 0000000..e343035
    --- /dev/null
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
    @@ -0,0 +1,61 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.metastore.hbase;
    +
    +import org.apache.commons.lang.StringUtils;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.junit.AfterClass;
    +import org.junit.Assert;
    +import org.junit.BeforeClass;
    +import org.junit.Test;
    +
    +import java.io.ByteArrayOutputStream;
    +import java.io.PrintStream;
    +
    +/**
    + * This is in a separate class because install tests shouldn't set up the metastore first.
    + */
    +public class TestHBaseSchemaTool2 extends HBaseIntegrationTests {
    + private static final Log LOG = LogFactory.getLog(TestHBaseSchemaTool.class.getName());
    + private String lsep = System.getProperty("line.separator");
    +
    + @BeforeClass
    + public static void startup() throws Exception {
    + HBaseIntegrationTests.startMiniCluster();
    + }
    +
    + @AfterClass
    + public static void shutdown() throws Exception {
    + HBaseIntegrationTests.shutdownMiniCluster();
    + }
    +
    + @Test
    + public void install() {
    + ByteArrayOutputStream outStr = new ByteArrayOutputStream();
    + PrintStream out = new PrintStream(outStr);
    + ByteArrayOutputStream errStr = new ByteArrayOutputStream();
    + PrintStream err = new PrintStream(errStr);
    +
    + HBaseSchemaTool tool = new HBaseSchemaTool();
    + tool.install(conf, err);
    + tool.go(true, null, null, null, conf, out, err);
    + Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep,
    + outStr.toString());
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/01580af2/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
    index 1ac060b..2fb3e8f 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
    @@ -57,8 +57,14 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
      import org.apache.hadoop.hive.metastore.api.Table;
      import org.apache.hadoop.hive.metastore.hbase.PartitionKeyComparator.Operator;
      import org.apache.hive.common.util.BloomFilter;
    +import org.apache.thrift.TBase;
    +import org.apache.thrift.TException;
    +import org.apache.thrift.protocol.TProtocol;
    +import org.apache.thrift.protocol.TSimpleJSONProtocol;
    +import org.apache.thrift.transport.TMemoryBuffer;

      import java.io.IOException;
    +import java.io.UnsupportedEncodingException;
      import java.nio.ByteBuffer;
      import java.security.MessageDigest;
      import java.security.NoSuchAlgorithmException;
    @@ -71,6 +77,7 @@ import java.util.HashSet;
      import java.util.Iterator;
      import java.util.List;
      import java.util.Map;
    +import java.util.NavigableMap;
      import java.util.Set;


    @@ -79,30 +86,29 @@ import java.util.Set;
       */
      public class HBaseReadWrite {

    - @VisibleForTesting final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS";
    - @VisibleForTesting final static String DB_TABLE = "HBMS_DBS";
    - @VisibleForTesting final static String FUNC_TABLE = "HBMS_FUNCS";
    - @VisibleForTesting final static String GLOBAL_PRIVS_TABLE = "HBMS_GLOBAL_PRIVS";
    - @VisibleForTesting final static String PART_TABLE = "HBMS_PARTITIONS";
    - @VisibleForTesting final static String ROLE_TABLE = "HBMS_ROLES";
    - @VisibleForTesting final static String SD_TABLE = "HBMS_SDS";
    - @VisibleForTesting final static String SECURITY_TABLE = "HBMS_SECURITY";
    - @VisibleForTesting final static String SEQUENCES_TABLE = "HBMS_SEQUENCES";
    - @VisibleForTesting final static String TABLE_TABLE = "HBMS_TBLS";
    - @VisibleForTesting final static String USER_TO_ROLE_TABLE = "HBMS_USER_TO_ROLE";
    - @VisibleForTesting final static String FILE_METADATA_TABLE = "HBMS_FILE_METADATA";
    - @VisibleForTesting final static byte[] CATALOG_CF = "c".getBytes(HBaseUtils.ENCODING);
    - @VisibleForTesting final static byte[] STATS_CF = "s".getBytes(HBaseUtils.ENCODING);
    - @VisibleForTesting final static String NO_CACHE_CONF = "no.use.cache";
    + final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS";
    + final static String DB_TABLE = "HBMS_DBS";
    + final static String FUNC_TABLE = "HBMS_FUNCS";
    + final static String GLOBAL_PRIVS_TABLE = "HBMS_GLOBAL_PRIVS";
    + final static String PART_TABLE = "HBMS_PARTITIONS";
    + final static String ROLE_TABLE = "HBMS_ROLES";
    + final static String SD_TABLE = "HBMS_SDS";
    + final static String SECURITY_TABLE = "HBMS_SECURITY";
    + final static String SEQUENCES_TABLE = "HBMS_SEQUENCES";
    + final static String TABLE_TABLE = "HBMS_TBLS";
    + final static String USER_TO_ROLE_TABLE = "HBMS_USER_TO_ROLE";
    + final static String FILE_METADATA_TABLE = "HBMS_FILE_METADATA";
    + final static byte[] CATALOG_CF = "c".getBytes(HBaseUtils.ENCODING);
    + final static byte[] STATS_CF = "s".getBytes(HBaseUtils.ENCODING);
    + final static String NO_CACHE_CONF = "no.use.cache";
        /**
         * List of tables in HBase
         */
    - public final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE, GLOBAL_PRIVS_TABLE,
    - PART_TABLE, USER_TO_ROLE_TABLE, ROLE_TABLE, SD_TABLE,
    - SECURITY_TABLE, SEQUENCES_TABLE, TABLE_TABLE,
    - FILE_METADATA_TABLE };
    - public final static Map<String, List<byte[]>> columnFamilies =
    - new HashMap<String, List<byte[]>> (tableNames.length);
    + public final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE,
    + GLOBAL_PRIVS_TABLE, PART_TABLE, USER_TO_ROLE_TABLE,
    + ROLE_TABLE, SD_TABLE, SECURITY_TABLE, SEQUENCES_TABLE,
    + TABLE_TABLE, FILE_METADATA_TABLE };
    + public final static Map<String, List<byte[]>> columnFamilies = new HashMap<> (tableNames.length);

        static {
          columnFamilies.put(AGGR_STATS_TABLE, Arrays.asList(CATALOG_CF));
    @@ -120,18 +126,14 @@ public class HBaseReadWrite {
          columnFamilies.put(FILE_METADATA_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
        }

    - /**
    - * Stores the bloom filter for the aggregated stats, to determine what partitions are in this
    - * aggregate.
    - */
    - final static byte[] MASTER_KEY_SEQUENCE = "mk".getBytes(HBaseUtils.ENCODING);
        final static byte[] AGGR_STATS_BLOOM_COL = "b".getBytes(HBaseUtils.ENCODING);
    + private final static byte[] AGGR_STATS_STATS_COL = "s".getBytes(HBaseUtils.ENCODING);
    + final static byte[] MASTER_KEY_SEQUENCE = "mk".getBytes(HBaseUtils.ENCODING);
        private final static byte[] CATALOG_COL = "c".getBytes(HBaseUtils.ENCODING);
        private final static byte[] ROLES_COL = "roles".getBytes(HBaseUtils.ENCODING);
        private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING);
        private final static byte[] DELEGATION_TOKEN_COL = "dt".getBytes(HBaseUtils.ENCODING);
        private final static byte[] MASTER_KEY_COL = "mk".getBytes(HBaseUtils.ENCODING);
    - private final static byte[] AGGR_STATS_STATS_COL = "s".getBytes(HBaseUtils.ENCODING);
        private final static byte[] GLOBAL_PRIVS_KEY = "gp".getBytes(HBaseUtils.ENCODING);
        private final static byte[] SEQUENCES_KEY = "seq".getBytes(HBaseUtils.ENCODING);
        private final static int TABLES_TO_CACHE = 10;
    @@ -205,6 +207,10 @@ public class HBaseReadWrite {
          return self.get();
        }

    + public Configuration getConf() {
    + return conf;
    + }
    +
        private HBaseReadWrite(Configuration configuration) {
          conf = configuration;
          HBaseConfiguration.addHbaseResources(conf);
    @@ -386,6 +392,35 @@ public class HBaseReadWrite {
          delete(DB_TABLE, key, null, null);
        }

    + /**
    + * Print out the database. Intended for use by {@link org.apache.hadoop.hive.metastore.hbase.HBaseSchemaTool}
    + * @param name name of database to print
    + * @return string printout of database
    + */
    + String printDatabase(String name) throws IOException, TException {
    + Database db = getDb(name);
    + if (db == null) return noSuch(name, "database");
    + else return dumpThriftObject(db);
    + }
    +
    + /**
    + * Print out databases.
    + * @param regex regular to use to search for databases
    + * @return databases as a string, one each
    + * @throws IOException
    + * @throws TException
    + */
    + List<String> printDatabases(String regex) throws IOException, TException {
    + List<Database> dbs = scanDatabases(regex);
    + if (dbs.size() == 0) {
    + return noMatch(regex, "database");
    + } else {
    + List<String> lines = new ArrayList<>();
    + for (Database db : dbs) lines.add(dumpThriftObject(db));
    + return lines;
    + }
    + }
    +
        /**********************************************************************************************
         * Function related methods
         *********************************************************************************************/
    @@ -453,6 +488,41 @@ public class HBaseReadWrite {
          delete(FUNC_TABLE, key, null, null);
        }

    + /**
    + * Print out a function
    + * @param key key to get the function, must include dbname.
    + * @return string of the function
    + * @throws IOException
    + * @throws TException
    + */
    + String printFunction(String key) throws IOException, TException {
    + byte[] k = HBaseUtils.buildKey(key);
    + byte[] serialized = read(FUNC_TABLE, k, CATALOG_CF, CATALOG_COL);
    + if (serialized == null) return noSuch(key, "function");
    + Function func = HBaseUtils.deserializeFunction(k, serialized);
    + return dumpThriftObject(func);
    + }
    +
    + /**
    + * Print out functions
    + * @param regex regular expression to use in matching functions
    + * @return list of strings, one function each
    + * @throws IOException
    + * @throws TException
    + */
    + List<String> printFunctions(String regex) throws IOException, TException {
    + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
    + Iterator<Result> iter = scan(FUNC_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
    + List<String> lines = new ArrayList<>();
    + while (iter.hasNext()) {
    + Result result = iter.next();
    + lines.add(dumpThriftObject(HBaseUtils.deserializeFunction(result.getRow(),
    + result.getValue(CATALOG_CF, CATALOG_COL))));
    + }
    + if (lines.size() == 0) lines = noMatch(regex, "function");
    + return lines;
    + }
    +
        /**********************************************************************************************
         * Global privilege related methods
         *********************************************************************************************/
    @@ -479,6 +549,18 @@ public class HBaseReadWrite {
          store(GLOBAL_PRIVS_TABLE, key, CATALOG_CF, CATALOG_COL, serialized);
        }

    + /**
    + * Print out the global privileges.
    + * @return string containing the global privileges
    + * @throws IOException
    + * @throws TException
    + */
    + String printGlobalPrivs() throws IOException, TException {
    + PrincipalPrivilegeSet pps = getGlobalPrivs();
    + if (pps == null) return "No global privileges";
    + else return dumpThriftObject(pps);
    + }
    +
        /**********************************************************************************************
         * Partition related methods
         *********************************************************************************************/
    @@ -645,7 +727,8 @@ public class HBaseReadWrite {
          }
          byte[] keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName, new ArrayList<String>(),
              new ArrayList<String>(), false);
    - List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix), -1, null);
    + List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, keyPrefix,
    + HBaseUtils.getEndPrefix(keyPrefix), -1, null);
          partCache.put(dbName, tableName, parts, true);
          return maxPartitions < parts.size() ? parts.subList(0, maxPartitions) : parts;
        }
    @@ -670,6 +753,206 @@ public class HBaseReadWrite {
         */
        List<Partition> scanPartitions(String dbName, String tableName, List<String> partVals,
                                       int maxPartitions) throws IOException, NoSuchObjectException {
    +
    + PartitionScanInfo psi = scanPartitionsInternal(dbName, tableName, partVals, maxPartitions);
    + List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, psi.keyPrefix,
    + psi.endKeyPrefix, maxPartitions, psi.filter);
    + partCache.put(dbName, tableName, parts, false);
    + return parts;
    + }
    +
    + List<Partition> scanPartitions(String dbName, String tableName, byte[] keyStart, byte[] keyEnd,
    + Filter filter, int maxPartitions)
    + throws IOException, NoSuchObjectException {
    + byte[] startRow = keyStart;
    + byte[] endRow;
    + if (keyEnd == null || keyEnd.length == 0) {
    + // stop when current db+table entries are over
    + endRow = HBaseUtils.getEndPrefix(startRow);
    + } else {
    + endRow = keyEnd;
    + }
    +
    + if (LOG.isDebugEnabled()) {
    + LOG.debug("Scanning partitions with start row <" + new String(startRow) + "> and end row <"
    + + new String(endRow) + ">");
    + }
    + return scanPartitionsWithFilter(dbName, tableName, startRow, endRow, maxPartitions, filter);
    + }
    +
    + /**
    + * Delete a partition
    + * @param dbName database name that table is in
    + * @param tableName table partition is in
    + * @param partVals partition values that define this partition, in the same order as the
    + * partition columns they are values for
    + * @throws IOException
    + */
    + void deletePartition(String dbName, String tableName, List<String> partTypes,
    + List<String> partVals) throws IOException {
    + deletePartition(dbName, tableName, partTypes, partVals, true);
    + }
    +
    + /**
    + * Print out a partition.
    + * @param partKey The key for the partition. This must include dbname.tablename._partkeys_
    + * where _partkeys_ is a dot separated list of partition values in the proper
    + * order.
    + * @return string containing the partition
    + * @throws IOException
    + * @throws TException
    + */
    + String printPartition(String partKey) throws IOException, TException {
    + // First figure out the table and fetch it
    + String[] partKeyParts = partKey.split(HBaseUtils.KEY_SEPARATOR_STR);
    + if (partKeyParts.length < 3) return noSuch(partKey, "partition");
    + Table table = getTable(partKeyParts[0], partKeyParts[1]);
    + if (table == null) return noSuch(partKey, "partition");
    +
    + byte[] key = HBaseUtils.buildPartitionKey(partKeyParts[0], partKeyParts[1],
    + HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys()),
    + Arrays.asList(Arrays.copyOfRange(partKeyParts, 2, partKeyParts.length)));
    + @SuppressWarnings("deprecation")
    + HTableInterface htab = conn.getHBaseTable(PART_TABLE);
    + Get g = new Get(key);
    + g.addColumn(CATALOG_CF, CATALOG_COL);
    + g.addFamily(STATS_CF);
    + Result result = htab.get(g);
    + if (result.isEmpty()) return noSuch(partKey, "partition");
    + return printOnePartition(result);
    + }
    +
    + /**
    + * Print partitions
    + * @param partKey a partial partition key. This must match the beginings of the partition key.
    + * It can be just dbname.tablename, or dbname.table.pval... where pval are the
    + * partition values in order. They must be in the correct order and they must
    + * be literal values (no regular expressions)
    + * @return partitions as strings
    + * @throws IOException
    + * @throws TException
    + */
    + List<String> printPartitions(String partKey) throws IOException, TException {
    + // First figure out the table and fetch it
    + // Split on dot here rather than the standard separator because this will be passed in as a
    + // regex, even though we aren't fully supporting regex's.
    + String[] partKeyParts = partKey.split("\\.");
    + if (partKeyParts.length < 2) return noMatch(partKey, "partition");
    + List<String> partVals = partKeyParts.length == 2 ? Arrays.asList("*") :
    + Arrays.asList(Arrays.copyOfRange(partKeyParts, 2, partKeyParts.length));
    + PartitionScanInfo psi;
    + try {
    + psi =
    + scanPartitionsInternal(partKeyParts[0], partKeyParts[1], partVals, -1);
    + } catch (NoSuchObjectException e) {
    + return noMatch(partKey, "partition");
    + }
    +
    + @SuppressWarnings("deprecation")
    + HTableInterface htab = conn.getHBaseTable(PART_TABLE);
    + Scan scan = new Scan();
    + scan.addColumn(CATALOG_CF, CATALOG_COL);
    + scan.addFamily(STATS_CF);
    + scan.setStartRow(psi.keyPrefix);
    + scan.setStopRow(psi.endKeyPrefix);
    + scan.setFilter(psi.filter);
    + Iterator<Result> iter = htab.getScanner(scan).iterator();
    + if (!iter.hasNext()) return noMatch(partKey, "partition");
    + List<String> lines = new ArrayList<>();
    + while (iter.hasNext()) {
    + lines.add(printOnePartition(iter.next()));
    + }
    + return lines;
    + }
    +
    + private String printOnePartition(Result result) throws IOException, TException {
    + byte[] key = result.getRow();
    + HBaseUtils.StorageDescriptorParts sdParts =
    + HBaseUtils.deserializePartition(key, result.getValue(CATALOG_CF, CATALOG_COL), this);
    + StringBuilder builder = new StringBuilder();
    + builder.append(dumpThriftObject(sdParts.containingPartition))
    + .append(" sdHash: ")
    + .append(Base64.encodeBase64URLSafeString(sdParts.sdHash))
    + .append(" stats:");
    + NavigableMap<byte[], byte[]> statsCols = result.getFamilyMap(STATS_CF);
    + for (Map.Entry<byte[], byte[]> statsCol : statsCols.entrySet()) {
    + builder.append(" column ")
    + .append(new String(statsCol.getKey(), HBaseUtils.ENCODING))
    + .append(": ");
    + ColumnStatistics pcs = buildColStats(key, false);
    + ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue());
    + builder.append(dumpThriftObject(cso));
    + }
    + return builder.toString();
    + }
    +
    + private void deletePartition(String dbName, String tableName, List<String> partTypes,
    + List<String> partVals, boolean decrementRefCnt) throws IOException {
    + // Find the partition so I can get the storage descriptor and drop it
    + partCache.remove(dbName, tableName, partVals);
    + if (decrementRefCnt) {
    + Partition p = getPartition(dbName, tableName, partVals, false);
    + decrementStorageDescriptorRefCount(p.getSd());
    + }
    + byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partTypes, partVals);
    + delete(PART_TABLE, key, null, null);
    + }
    +
    + private Partition getPartition(String dbName, String tableName, List<String> partVals,
    + boolean populateCache) throws IOException {
    + Partition cached = partCache.get(dbName, tableName, partVals);
    + if (cached != null) return cached;
    + byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName,
    + HBaseUtils.getPartitionKeyTypes(getTable(dbName, tableName).getPartitionKeys()), partVals);
    + byte[] serialized = read(PART_TABLE, key, CATALOG_CF, CATALOG_COL);
    + if (serialized == null) return null;
    + HBaseUtils.StorageDescriptorParts sdParts =
    + HBaseUtils.deserializePartition(dbName, tableName, partVals, serialized);
    + StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
    + HBaseUtils.assembleStorageDescriptor(sd, sdParts);
    + if (populateCache) partCache.put(dbName, tableName, sdParts.containingPartition);
    + return sdParts.containingPartition;
    + }
    +
    +
    + private static class PartitionScanInfo {
    + final String dbName;
    + final String tableName;
    + final byte[] keyPrefix;
    + final byte[] endKeyPrefix;
    + final int maxPartitions;
    + final Filter filter;
    +
    + PartitionScanInfo(String d, String t, byte[] k, byte[] e, int m, Filter f) {
    + dbName = d;
    + tableName = t;
    + keyPrefix = k;
    + endKeyPrefix = e;
    + maxPartitions = m;
    + filter = f;
    + }
    +
    + @Override
    + public String toString() {
    + return new StringBuilder("dbName:")
    + .append(dbName)
    + .append(" tableName:")
    + .append(tableName)
    + .append(" keyPrefix:")
    + .append(Base64.encodeBase64URLSafeString(keyPrefix))
    + .append(" endKeyPrefix:")
    + .append(Base64.encodeBase64URLSafeString(endKeyPrefix))
    + .append(" maxPartitions:")
    + .append(maxPartitions)
    + .append(" filter:")
    + .append(filter.toString())
    + .toString();
    + }
    + }
    +
    + private PartitionScanInfo scanPartitionsInternal(String dbName, String tableName,
    + List<String> partVals, int maxPartitions)
    + throws IOException, NoSuchObjectException {
          // First, build as much of the key as we can so that we make the scan as tight as possible.
          List<String> keyElements = new ArrayList<>();
          keyElements.add(dbName);
    @@ -700,7 +983,7 @@ public class HBaseReadWrite {
          }
          keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName,
              HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys().subList(0, keyElements.size()-2)),
    - keyElements.subList(2, keyElements.size()));
    + keyElements.subList(2, keyElements.size()));

          // Now, build a filter out of the remaining keys
          List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
    @@ -739,76 +1022,13 @@ public class HBaseReadWrite {
                filter + ">");
          }

    - List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, keyPrefix,
    - HBaseUtils.getEndPrefix(keyPrefix), maxPartitions, filter);
    - partCache.put(dbName, tableName, parts, false);
    - return parts;
    - }
    -
    - List<Partition> scanPartitions(String dbName, String tableName, byte[] keyStart, byte[] keyEnd,
    - Filter filter, int maxPartitions) throws IOException, NoSuchObjectException {
    - byte[] startRow = keyStart;
    - byte[] endRow;
    - if (keyEnd == null || keyEnd.length == 0) {
    - // stop when current db+table entries are over
    - endRow = HBaseUtils.getEndPrefix(startRow);
    - } else {
    - endRow = keyEnd;
    - }
    -
    - if (LOG.isDebugEnabled()) {
    - LOG.debug("Scanning partitions with start row <" + new String(startRow) + "> and end row <"
    - + new String(endRow) + ">");
    - }
    - return scanPartitionsWithFilter(dbName, tableName, startRow, endRow, maxPartitions, filter);
    - }
    -
    -
    -
    - /**
    - * Delete a partition
    - * @param dbName database name that table is in
    - * @param tableName table partition is in
    - * @param partVals partition values that define this partition, in the same order as the
    - * partition columns they are values for
    - * @throws IOException
    - */
    - void deletePartition(String dbName, String tableName, List<String> partTypes,
    - List<String> partVals) throws IOException {
    - deletePartition(dbName, tableName, partTypes, partVals, true);
    - }
    -
    - private void deletePartition(String dbName, String tableName, List<String> partTypes,
    - List<String> partVals, boolean decrementRefCnt) throws IOException {
    - // Find the partition so I can get the storage descriptor and drop it
    - partCache.remove(dbName, tableName, partVals);
    - if (decrementRefCnt) {
    - Partition p = getPartition(dbName, tableName, partVals, false);
    - decrementStorageDescriptorRefCount(p.getSd());
    - }
    - byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partTypes, partVals);
    - delete(PART_TABLE, key, null, null);
    - }
    -
    - private Partition getPartition(String dbName, String tableName, List<String> partVals,
    - boolean populateCache) throws IOException {
    - Partition cached = partCache.get(dbName, tableName, partVals);
    - if (cached != null) return cached;
    - byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName,
    - HBaseUtils.getPartitionKeyTypes(getTable(dbName, tableName).getPartitionKeys()), partVals);
    - byte[] serialized = read(PART_TABLE, key, CATALOG_CF, CATALOG_COL);
    - if (serialized == null) return null;
    - HBaseUtils.StorageDescriptorParts sdParts =
    - HBaseUtils.deserializePartition(dbName, tableName, partVals, serialized);
    - StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
    - HBaseUtils.assembleStorageDescriptor(sd, sdParts);
    - if (populateCache) partCache.put(dbName, tableName, sdParts.containingPartition);
    - return sdParts.containingPartition;
    + return new PartitionScanInfo(dbName, tableName, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
    + maxPartitions, filter);
        }

        private List<Partition> scanPartitionsWithFilter(String dbName, String tableName,
    - byte[] startRow, byte [] endRow, int maxResults, Filter filter)
    - throws IOException {
    + byte[] startRow, byte [] endRow, int maxResults,
    + Filter filter) throws IOException {
          Iterator<Result> iter =
              scan(PART_TABLE, startRow, endRow, CATALOG_CF, CATALOG_COL, filter);
          List<FieldSchema> tablePartitions = getTable(dbName, tableName).getPartitionKeys();
    @@ -817,7 +1037,7 @@ public class HBaseReadWrite {
          for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
            Result result = iter.next();
            HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializePartition(dbName, tableName,
    - tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL), staticConf);
    + tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL), conf);
            StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
            HBaseUtils.assembleStorageDescriptor(sd, sdParts);
            parts.add(sdParts.containingPartition);
    @@ -1158,14 +1378,7 @@ public class HBaseReadWrite {
         * @throws IOException
         */
        List<Role> scanRoles() throws IOException {
    - Iterator<Result> iter = scan(ROLE_TABLE, CATALOG_CF, CATALOG_COL);
    - List<Role> roles = new ArrayList<>();
    - while (iter.hasNext()) {
    - Result result = iter.next();
    - roles.add(HBaseUtils.deserializeRole(result.getRow(),
    - result.getValue(CATALOG_CF, CATALOG_COL)));
    - }
    - return roles;
    + return scanRoles(null);
        }

        /**
    @@ -1189,6 +1402,70 @@ public class HBaseReadWrite {
          roleCache.remove(roleName);
        }

    + String printRolesForUser(String userName) throws IOException {
    + List<String> roles = getUserRoles(userName);
    + if (roles == null || roles.size() == 0) return noSuch(userName, "user");
    + return org.apache.commons.lang.StringUtils.join(roles, ',');
    + }
    +
    + List<String> printRolesForUsers(String regex) throws IOException {
    + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
    + Iterator<Result> iter = scan(USER_TO_ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
    + List<String> lines = new ArrayList<>();
    + while (iter.hasNext()) {
    + Result result = iter.next();
    + lines.add(new String(result.getRow(), HBaseUtils.ENCODING) + ": " +
    + org.apache.commons.lang.StringUtils.join(
    + HBaseUtils.deserializeRoleList(result.getValue(CATALOG_CF, CATALOG_COL)), ','));
    + }
    + if (lines.size() == 0) lines = noMatch(regex, "user");
    + return lines;
    + }
    +
    + /**
    + * Print out a role
    + * @param name name of role to print
    + * @return string printout of role
    + */
    + String printRole(String name) throws IOException, TException {
    + Role role = getRole(name);
    + if (role == null) return noSuch(name, "role");
    + else return dumpThriftObject(role);
    + }
    +
    + /**
    + * Print out roles.
    + * @param regex regular to use to search for roles
    + * @return string printout of roles
    + * @throws IOException
    + * @throws TException
    + */
    + List<String> printRoles(String regex) throws IOException, TException {
    + List<Role> roles = scanRoles(regex);
    + if (roles.size() == 0) {
    + return noMatch(regex, "role");
    + } else {
    + List<String> lines = new ArrayList<>();
    + for (Role role : roles) lines.add(dumpThriftObject(role));
    + return lines;
    + }
    + }
    +
    + private List<Role> scanRoles(String regex) throws IOException {
    + Filter filter = null;
    + if (regex != null) {
    + filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
    + }
    + Iterator<Result> iter = scan(ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
    + List<Role> roles = new ArrayList<>();
    + while (iter.hasNext()) {
    + Result result = iter.next();
    + roles.add(HBaseUtils.deserializeRole(result.getRow(),
    + result.getValue(CATALOG_CF, CATALOG_COL)));
    + }
    + return roles;
    + }
    +
        private void buildRoleCache() throws IOException {
          if (!entireRoleTableInCache) {
            Iterator<Result> roles = scan(ROLE_TABLE, CATALOG_CF, ROLES_COL);
    @@ -1350,6 +1627,71 @@ public class HBaseReadWrite {
          deleteTable(dbName, tableName, true);
        }

    + /**
    + * Print out a table.
    + * @param name The name for the table. This must include dbname.tablename
    + * @return string containing the table
    + * @throws IOException
    + * @throws TException
    + */
    + String printTable(String name) throws IOException, TException {
    + byte[] key = HBaseUtils.buildKey(name);
    + @SuppressWarnings("deprecation")
    + HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
    + Get g = new Get(key);
    + g.addColumn(CATALOG_CF, CATALOG_COL);
    + g.addFamily(STATS_CF);
    + Result result = htab.get(g);
    + if (result.isEmpty()) return noSuch(name, "table");
    + return printOneTable(result);
    + }
    +
    + /**
    + * Print tables
    + * @param regex to use to find the tables. Remember that dbname is in each
    + * table name.
    + * @return tables as strings
    + * @throws IOException
    + * @throws TException
    + */
    + List<String> printTables(String regex) throws IOException, TException {
    + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
    + @SuppressWarnings("deprecation")
    + HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
    + Scan scan = new Scan();
    + scan.addColumn(CATALOG_CF, CATALOG_COL);
    + scan.addFamily(STATS_CF);
    + scan.setFilter(filter);
    + Iterator<Result> iter = htab.getScanner(scan).iterator();
    + if (!iter.hasNext()) return noMatch(regex, "table");
    + List<String> lines = new ArrayList<>();
    + while (iter.hasNext()) {
    + lines.add(printOneTable(iter.next()));
    + }
    + return lines;
    + }
    +
    + private String printOneTable(Result result) throws IOException, TException {
    + byte[] key = result.getRow();
    + HBaseUtils.StorageDescriptorParts sdParts =
    + HBaseUtils.deserializeTable(key, result.getValue(CATALOG_CF, CATALOG_COL));
    + StringBuilder builder = new StringBuilder();
    + builder.append(dumpThriftObject(sdParts.containingTable))
    + .append(" sdHash: ")
    + .append(Base64.encodeBase64URLSafeString(sdParts.sdHash))
    + .append(" stats:");
    + NavigableMap<byte[], byte[]> statsCols = result.getFamilyMap(STATS_CF);
    + for (Map.Entry<byte[], byte[]> statsCol : statsCols.entrySet()) {
    + builder.append(" column ")
    + .append(new String(statsCol.getKey(), HBaseUtils.ENCODING))
    + .append(": ");
    + ColumnStatistics pcs = buildColStats(key, true);
    + ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue());
    + builder.append(dumpThriftObject(cso));
    + }
    + return builder.toString();
    + }
    +
        private void deleteTable(String dbName, String tableName, boolean decrementRefCnt)
            throws IOException {
          tableCache.remove(new ObjectPair<>(dbName, tableName));
    @@ -1466,6 +1808,37 @@ public class HBaseReadWrite {
          return key;
        }

    + /**
    + * Print out a storage descriptor.
    + * @param hash hash that is the key of the storage descriptor
    + * @return string version of the storage descriptor
    + */
    + String printStorageDescriptor(byte[] hash) throws IOException, TException {
    + byte[] serialized = read(SD_TABLE, hash, CATALOG_CF, CATALOG_COL);
    + if (serialized == null) return noSuch(Base64.encodeBase64URLSafeString(hash), "storage descriptor");
    + return dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(serialized));
    + }
    +
    + /**
    + * Print all of the storage descriptors. This doesn't take a regular expression since the key
    + * is an md5 hash and it's hard to see how a regex on this would be useful.
    + * @return list of all storage descriptors as strings
    + * @throws IOException
    + * @throws TException
    + */
    + List<String> printStorageDescriptors() throws IOException, TException {
    + Iterator<Result> results = scan(SD_TABLE, CATALOG_CF, CATALOG_COL);
    + if (!results.hasNext()) return Arrays.asList("No storage descriptors");
    + List<String> lines = new ArrayList<>();
    + while (results.hasNext()) {
    + Result result = results.next();
    + lines.add(Base64.encodeBase64URLSafeString(result.getRow()) + ": " +
    + dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(result.getValue(CATALOG_CF,
    + CATALOG_COL))));
    + }
    + return lines;
    + }
    +
        private static class ByteArrayWrapper {
          byte[] wrapped;

    @@ -1604,25 +1977,9 @@ public class HBaseReadWrite {
                if (colStats == null) {
                  // We initialize this late so that we don't create extras in the case of
                  // partitions with no stats
    - colStats = new ColumnStatistics();
    + colStats = buildColStats(results[i].getRow(), false);
                  statsList.add(colStats);
    - ColumnStatisticsDesc csd = new ColumnStatisticsDesc();
    -
    - // We need to figure out which partition these call stats are from. To do that we
    - // recontruct the key. We have to pull the dbName and tableName out of the key to
    - // find the partition values.
    - byte[] key = results[i].getRow();
    - List<String> reconstructedPartVals =
    - HBaseUtils.deserializePartitionKey(getTable(dbName, tblName).getPartitionKeys(), key,
    - staticConf);
    - String partName = valToPartMap.get(reconstructedPartVals);
    - assert partName != null;
    - csd.setIsTblLevel(false);
    - csd.setDbName(dbName);
    - csd.setTableName(tblName);
    - csd.setPartName(partName);
    - colStats.setStatsDesc(csd);
    - }
    + }
                ColumnStatisticsObj cso =
                    HBaseUtils.deserializeStatsForOneColumn(colStats, serializedColStats);
                cso.setColName(colNames.get(j));
    @@ -1729,6 +2086,36 @@ public class HBaseReadWrite {
          return partVals == null ? TABLE_TABLE : PART_TABLE;
        }

    + private ColumnStatistics buildColStats(byte[] key, boolean fromTable) throws IOException {
    + // We initialize this late so that we don't create extras in the case of
    + // partitions with no stats
    + ColumnStatistics colStats = new ColumnStatistics();
    + ColumnStatisticsDesc csd = new ColumnStatisticsDesc();
    +
    + // If this is a table key, parse it as one
    + List<String> reconstructedKey;
    + if (fromTable) {
    + reconstructedKey = Arrays.asList(HBaseUtils.deserializeKey(key));
    + csd.setIsTblLevel(true);
    + } else {
    + reconstructedKey = HBaseUtils.deserializePartitionKey(key, this);
    + csd.setIsTblLevel(false);
    + }
    + csd.setDbName(reconstructedKey.get(0));
    + csd.setTableName(reconstructedKey.get(1));
    + if (!fromTable) {
    + // Build the part name, for which we need the table
    + Table table = getTable(reconstructedKey.get(0), reconstructedKey.get(1));
    + if (table == null) {
    + throw new RuntimeException("Unable to find table " + reconstructedKey.get(0) + "." +
    + reconstructedKey.get(1) + " even though I have a partition for it!");
    + }
    + csd.setPartName(HBaseStore.buildExternalPartName(table, reconstructedKey.subList(2,
    + reconstructedKey.size())));
    + }
    + colStats.setStatsDesc(csd);
    + return colStats;
    + }
        /**********************************************************************************************
         * File metadata related methods
         *********************************************************************************************/
    @@ -1881,6 +2268,35 @@ public class HBaseReadWrite {
          delete(SECURITY_TABLE, key, CATALOG_CF, MASTER_KEY_COL);
        }

    + /**
    + * One method to print all rows in the security table. It's not expected to be large.
    + * @return each row as one string
    + * @throws IOException
    + */
    + List<String> printSecurity() throws IOException {
    + HTableInterface htab = conn.getHBaseTable(SECURITY_TABLE);
    + Scan scan = new Scan();
    + scan.addColumn(CATALOG_CF, MASTER_KEY_COL);
    + scan.addColumn(CATALOG_CF, DELEGATION_TOKEN_COL);
    + Iterator<Result> iter = htab.getScanner(scan).iterator();
    + if (!iter.hasNext()) return Arrays.asList("No security related entries");
    + List<String> lines = new ArrayList<>();
    + while (iter.hasNext()) {
    + Result result = iter.next();
    + byte[] val = result.getValue(CATALOG_CF, MASTER_KEY_COL);
    + if (val != null) {
    + int seqNo = Integer.valueOf(new String(result.getRow(), HBaseUtils.ENCODING));
    + lines.add("Master key " + seqNo + ": " + HBaseUtils.deserializeMasterKey(val));
    + } else {
    + val = result.getValue(CATALOG_CF, DELEGATION_TOKEN_COL);
    + if (val == null) throw new RuntimeException("Huh? No master key, no delegation token!");
    + lines.add("Delegation token " + new String(result.getRow(), HBaseUtils.ENCODING) + ": " +
    + HBaseUtils.deserializeDelegationToken(val));
    + }
    + }
    + return lines;
    + }
    +
        /**********************************************************************************************
         * Sequence methods
         *********************************************************************************************/
    @@ -1896,6 +2312,25 @@ public class HBaseReadWrite {
          return val;
        }

    + /**
    + * One method to print all entries in the sequence table. It's not expected to be large.
    + * @return each sequence as one string
    + * @throws IOException
    + */
    + List<String> printSequences() throws IOException {
    + HTableInterface htab = conn.getHBaseTable(SEQUENCES_TABLE);
    + Get g = new Get(SEQUENCES_KEY);
    + g.addFamily(CATALOG_CF);
    + Result result = htab.get(g);
    + if (result.isEmpty()) return Arrays.asList("No sequences");
    + List<String> lines = new ArrayList<>();
    + for (Map.Entry<byte[], byte[]> entry : result.getFamilyMap(CATALOG_CF).entrySet()) {
    + lines.add(new String(entry.getKey(), HBaseUtils.ENCODING) + ": " +
    + new String(entry.getValue(), HBaseUtils.ENCODING));
    + }
    + return lines;
    + }
    +
        /**********************************************************************************************
         * Cache methods
         *********************************************************************************************/
    @@ -2055,12 +2490,27 @@ public class HBaseReadWrite {
          return scanner.iterator();
        }

    + /**********************************************************************************************
    + * Printing methods
    + *********************************************************************************************/
    + private String noSuch(String name, String type) {
    + return "No such " + type + ": " + name.replaceAll(HBaseUtils.KEY_SEPARATOR_STR, ".");
    + }

    + private List<String> noMatch(String regex, String type) {
    + return Arrays.asList("No matching " + type + ": " + regex);
    + }
    +
    + private String dumpThriftObject(TBase obj) throws TException, UnsupportedEncodingException {
    + TMemoryBuffer buf = new TMemoryBuffer(1000);
    + TProtocol protocol = new TSimpleJSONProtocol(buf);
    + obj.write(protocol);
    + return buf.toString("UTF-8");
    + }

        /**********************************************************************************************
         * Testing methods and classes
         *********************************************************************************************/
    -
        @VisibleForTesting
        int countStorageDescriptor() throws IOException {
          ResultScanner scanner = conn.getHBaseTable(SD_TABLE).getScanner(new Scan());

    http://git-wip-us.apache.org/repos/asf/hive/blob/01580af2/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
    index 8eb6116..567970c 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
    @@ -18,27 +18,20 @@
       */
      package org.apache.hadoop.hive.metastore.hbase;

    +import com.google.common.annotations.VisibleForTesting;
      import org.apache.commons.cli.CommandLine;
      import org.apache.commons.cli.GnuParser;
      import org.apache.commons.cli.HelpFormatter;
      import org.apache.commons.cli.OptionBuilder;
      import org.apache.commons.cli.Options;
    -import org.apache.commons.lang.StringUtils;
    +import org.apache.commons.cli.ParseException;
    +import org.apache.commons.codec.binary.Base64;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.conf.Configuration;
    -import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
    -import org.apache.hadoop.hive.metastore.api.Database;
    -import org.apache.hadoop.hive.metastore.api.Function;
    -import org.apache.hadoop.hive.metastore.api.Partition;
    -import org.apache.hadoop.hive.metastore.api.Role;
    -import org.apache.hadoop.hive.metastore.api.Table;
    -import org.apache.thrift.TBase;
    -import org.apache.thrift.TException;
    -import org.apache.thrift.protocol.TProtocol;
    -import org.apache.thrift.protocol.TSimpleJSONProtocol;
    -import org.apache.thrift.transport.TMemoryBuffer;
    -
    -import java.io.IOException;
    -import java.lang.reflect.Method;
    +
    +import java.io.PrintStream;
    +import java.util.ArrayList;
      import java.util.Arrays;
      import java.util.List;

    @@ -47,194 +40,157 @@ import java.util.List;
       */
      public class HBaseSchemaTool {

    - private static String[] commands = {"db", "part", "parts", "role", "table", "function",
    - "install"};
    + static final private Log LOG = LogFactory.getLog(HBaseReadWrite.class.getName());

    - public static void main(String[] args) throws Exception {
    + public static void main(String[] args) {
          Options options = new Options();

          options.addOption(OptionBuilder
    - .withLongOpt("column")
    - .withDescription("Comma separated list of column names")
    - .hasArg()
    - .create('c'));
    + .withLongOpt("help")
    + .withDescription("You're looking at it")
    + .create('h'));

          options.addOption(OptionBuilder
    - .withLongOpt("db")
    - .withDescription("Database name")
    - .hasArg()
    - .create('d'));
    + .withLongOpt("install")
    + .withDescription("Install the schema onto an HBase cluster.")
    + .create('i'));

          options.addOption(OptionBuilder
    - .withLongOpt("function")
    - .withDescription("Function name")
    + .withLongOpt("key")
    + .withDescription("Key to scan with. This should be an exact key (not a regular expression")
              .hasArg()
    - .create('f'));
    + .create('k'));

          options.addOption(OptionBuilder
    - .withLongOpt("help")
    - .withDescription("You're looking at it")
    - .create('h'));
    + .withLongOpt("list-tables")
    + .withDescription("List tables in HBase metastore")
    + .create('l'));

          options.addOption(OptionBuilder
    - .withLongOpt("role")
    - .withDescription("Role name")
    + .withLongOpt("regex-key")
    + .withDescription("Regular expression to scan keys with.")
              .hasArg()
              .create('r'));

          options.addOption(OptionBuilder
    - .withLongOpt("partvals")
    - .withDescription("Comma separated list of partition values, in order of partition columns")
    - .hasArg()
    - .create('p'));
    -
    - options.addOption(OptionBuilder
    - .withLongOpt("stats")
    - .withDescription("Get statistics rather than catalog object")
    - .create('s'));
    -
    - options.addOption(OptionBuilder
              .withLongOpt("table")
    - .withDescription("Table name")
    + .withDescription("HBase metastore table to scan")
              .hasArg()
              .create('t'));

    - CommandLine cli = new GnuParser().parse(options, args);
    + CommandLine cli = null;
    + try {
    + cli = new GnuParser().parse(options, args);
    + } catch (ParseException e) {
    + System.err.println("Parse Exception: " + e.getMessage());
    + usage(options);
    + return;
    + }

          if (cli.hasOption('h')) {
    - HelpFormatter formatter = new HelpFormatter();
    - formatter.printHelp("hbaseschematool", options);
    + usage(options);
            return;
          }

    - String[] cmds = cli.getArgs();
    - if (cmds.length != 1) {
    - System.err.print("Must include a cmd, valid cmds are: ");
    - for (int i = 0; i < commands.length; i++) {
    - if (i != 0) System.err.print(", ");
    - System.err.print(commands[i]);
    - }
    - System.err.println();
    - System.exit(1);
    - }
    - String cmd = cmds[0];
    + Configuration conf = new Configuration();

    - List<String> parts = null;
    - if (cli.hasOption('p')) {
    - parts = Arrays.asList(cli.getOptionValue('p').split(","));
    + if (cli.hasOption('i')) {
    + new HBaseSchemaTool().install(conf, System.err);
    + return;
          }

    - List<String> cols = null;
    - if (cli.hasOption('c')) {
    - cols = Arrays.asList(cli.getOptionValue('c').split(","));
    + String key = null;
    + if (cli.hasOption('k')) key = cli.getOptionValue('k');
    + String regex = null;
    + if (cli.hasOption('r')) regex = cli.getOptionValue('r');
    + if (key != null && regex != null) {
    + usage(options);
    + return;
          }
    + if (key == null && regex == null) regex = ".*";

    - HBaseSchemaTool tool = new HBaseSchemaTool(cli.getOptionValue('d'), cli.getOptionValue('t'),
    - parts, cli.getOptionValue('f'), cli.getOptionValue('r'), cols, cli.hasOption('s'));
    - Method method = tool.getClass().getMethod(cmd);
    - method.invoke(tool);
    -
    -
    + // I do this in the object rather than in the static main so that it's easier to test.
    + new HBaseSchemaTool().go(cli.hasOption('l'), cli.getOptionValue('t'), key, regex, conf,
    + System.out, System.err);
        }

    - private HBaseReadWrite hrw;
    - private String dbName;
    - private String funcName;
    - private String tableName;
    - private List<String> partVals;
    - private String roleName;
    - private List<String> colNames;
    - private boolean hasStats;
    -
    - private HBaseSchemaTool(String dbname, String tn, List<String> pv, String fn, String rn,
    - List<String> cn, boolean s) {
    - dbName = dbname;
    - tableName = tn;
    - partVals = pv;
    - funcName = fn;
    - roleName = rn;
    - colNames = cn;
    - hasStats = s;
    - HBaseReadWrite.setConf(new Configuration());
    - hrw = HBaseReadWrite.getInstance();
    + private static void usage(Options options) {
    + HelpFormatter formatter = new HelpFormatter();
    + String header = "This tool dumps contents of your hbase metastore. You need to specify\n" +
    + "the table to dump. You can optionally specify a regular expression on the key for\n" +
    + "the table. Keep in mind that the key is often a compound. For partitions regular\n" +
    + "expressions are not used because non-string values are\nstored in binary. Instead for " +
    + "partition you can specify as much of the exact prefix as you want. So you can give " +
    + "dbname.tablename or dbname.tablename.pval1...";
    + String footer = "If neither key or regex is provided a regex of .* will be assumed. You\n" +
    + "cannot set both key and regex.";
    + formatter.printHelp("hbaseschematool", header, options, footer);
    + return;
        }

    - public void db() throws IOException, TException {
    - Database db = hrw.getDb(dbName);
    - if (db == null) System.err.println("No such database: " + db);
    - else dump(db);
    - }
    -
    - public void install() throws IOException {
    - HBaseReadWrite.createTablesIfNotExist();
    - }
    -
    - public void part() throws IOException, TException {
    - if (hasStats) {
    - Table table = hrw.getTable(dbName, tableName);
    - if (table == null) {
    - System.err.println("No such table: " + dbName + "." + tableName);
    - return;
    - }
    - String partName = HBaseStore.buildExternalPartName(table, partVals);
    - List<ColumnStatistics> stats = hrw.getPartitionStatistics(dbName, tableName,
    - Arrays.asList(partName), Arrays.asList(partVals), colNames);
    - if (stats == null) {
    - System.err.println("No stats for " + dbName + "." + tableName + "." +
    - StringUtils.join(partVals, ':'));
    - } else {
    - for (ColumnStatistics stat : stats) dump(stat);
    - }
    + @VisibleForTesting void go(boolean listTables, String table, String key, String regex,
    + Configuration conf, PrintStream out, PrintStream err) {
    + List<String> lines = new ArrayList<>();
    + if (listTables) {
    + lines = Arrays.asList(HBaseReadWrite.tableNames);
          } else {
    - Partition part = hrw.getPartition(dbName, tableName, partVals);
    - if (part == null) {
    - System.err.println("No such partition: " + dbName + "." + tableName + "." +
    - StringUtils.join(partVals, ':'));
    - } else {
    - dump(part);
    + // If they've used '.' as a key separator we need to replace it with the separator used by
    + // HBaseUtils
    + if (key != null) key = key.replace('.', HBaseUtils.KEY_SEPARATOR);
    + try {
    + HBaseReadWrite.setConf(conf);
    + HBaseReadWrite hrw = HBaseReadWrite.getInstance();
    + if (table.equalsIgnoreCase(HBaseReadWrite.DB_TABLE)) {
    + if (key != null) lines.add(hrw.printDatabase(key));
    + else lines.addAll(hrw.printDatabases(regex));
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.FUNC_TABLE)) {
    + if (key != null) lines.add(hrw.printFunction(key));
    + else lines.addAll(hrw.printFunctions(regex));
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.GLOBAL_PRIVS_TABLE)) {
    + // Ignore whatever they passed, there's always only either one or zero global privileges
    + lines.add(hrw.printGlobalPrivs());
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.PART_TABLE)) {
    + if (key != null) lines.add(hrw.printPartition(key));
    + else lines.addAll(hrw.printPartitions(regex));
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.USER_TO_ROLE_TABLE)) {
    + if (key != null) lines.add(hrw.printRolesForUser(key));
    + else lines.addAll(hrw.printRolesForUsers(regex));
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.ROLE_TABLE)) {
    + if (key != null) lines.add(hrw.printRole(key));
    + else lines.addAll(hrw.printRoles(regex));
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.TABLE_TABLE)) {
    + if (key != null) lines.add(hrw.printTable(key));
    + else lines.addAll(hrw.printTables(regex));
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.SD_TABLE)) {
    + if (key != null) lines.add(hrw.printStorageDescriptor(Base64.decodeBase64(key)));
    + else lines.addAll(hrw.printStorageDescriptors());
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.SECURITY_TABLE)) {
    + // We always print all of security, we don't worry about finding particular entries.
    + lines.addAll(hrw.printSecurity());
    + } else if (table.equalsIgnoreCase(HBaseReadWrite.SEQUENCES_TABLE)) {
    + // We always print all of sequences, we don't worry about finding particular entries.
    + lines.addAll(hrw.printSequences());
    + } else {
    + err.println("Unknown table: " + table);
    + return;
    + }
    + } catch (Exception e) {
    + err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage());
    + return;
            }
          }
    + for (String line : lines) out.println(line);
        }

    - public void parts() throws IOException, TException {
    - List<Partition> parts = hrw.scanPartitionsInTable(dbName, tableName, -1);
    - if (parts == null) {
    - System.err.println("No such table: " + dbName + "." + tableName);
    - } else {
    - for (Partition p : parts) dump(p);
    - }
    - }
    -
    - public void role() throws IOException, TException {
    - Role role = hrw.getRole(roleName);
    - if (role == null) System.err.println("No such role: " + roleName);
    - else dump(role);
    - }
    -
    - public void table() throws IOException, TException {
    - if (hasStats) {
    - ColumnStatistics stats = hrw.getTableStatistics(dbName, tableName, colNames);
    - if (stats == null) System.err.println("No stats for " + dbName + "." + tableName);
    - else dump(stats);
    - } else {
    - Table table = hrw.getTable(dbName, tableName);
    - if (table == null) System.err.println("No such table: " + dbName + "." + tableName);
    - else dump(table);
    + @VisibleForTesting void install(Configuration conf, PrintStream err) {
    + try {
    + // We need to set the conf because createTablesIfNotExist will get a thread local version
    + // which requires that the configuration object be set.
    + HBaseReadWrite.setConf(conf);
    + HBaseReadWrite.createTablesIfNotExist();
    + } catch (Exception e) {
    + err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage());
    + return;
          }
        }
    -
    - public void function() throws IOException, TException {
    - Function func = hrw.getFunction(dbName, funcName);
    - if (func == null) System.err.println("No such function: " + dbName + "." + funcName);
    - else dump(func);
    - }
    -
    - private void dump(TBase thriftObj) throws TException {
    - TMemoryBuffer buf = new TMemoryBuffer(1000);
    - TProtocol protocol = new TSimpleJSONProtocol(buf);
    - thriftObj.write(protocol);
    - System.out.println(new String(buf.getArray()));
    - }
    -
    -
      }

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 28, '15 at 7:39p
activeOct 28, '15 at 7:39p
posts2
users1
websitehive.apache.org

1 user in discussion

Gates: 2 posts

People

Translate

site design / logo © 2021 Grokbase