FAQ
Repository: hive
Updated Branches:
   refs/heads/master 287f0451b -> 74197688f


HIVE-13527: Using deprecated APIs in HBase client causes zookeeper connection leaks (Naveen Gangam via Chaoyu Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/74197688
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/74197688
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/74197688

Branch: refs/heads/master
Commit: 74197688f18d9a6bee6393cfe1d7ab635beb3db5
Parents: 287f045
Author: ctang <ctang@cloudera.com>
Authored: Sat Apr 23 20:50:00 2016 -0400
Committer: ctang <ctang@cloudera.com>
Committed: Sat Apr 23 20:50:00 2016 -0400

----------------------------------------------------------------------
  .../hive/hbase/HiveHBaseTableInputFormat.java | 33 ++++++++++++++++++--
  1 file changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/74197688/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
index 88d1865..d9db624 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
@@ -26,6 +26,9 @@ import java.util.Map;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hbase.HBaseConfiguration;
  import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
  import org.apache.hadoop.hbase.client.HTable;
  import org.apache.hadoop.hbase.client.Result;
  import org.apache.hadoop.hbase.client.Scan;
@@ -84,6 +87,7 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase

    static final Logger LOG = LoggerFactory.getLogger(HiveHBaseTableInputFormat.class);
    private static final Object hbaseTableMonitor = new Object();
+ private Connection conn = null;

    @Override
    public RecordReader<ImmutableBytesWritable, ResultWritable> getRecordReader(
@@ -94,7 +98,11 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
      HBaseSplit hbaseSplit = (HBaseSplit) split;
      TableSplit tableSplit = hbaseSplit.getTableSplit();

- setHTable(HiveHBaseInputFormatUtil.getTable(jobConf));
+ if (conn == null) {
+ conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf));
+ }
+ initializeTable(conn, tableSplit.getTable());
+
      setScan(HiveHBaseInputFormatUtil.getScan(jobConf));

      Job job = new Job(jobConf);
@@ -107,6 +115,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
        recordReader.initialize(tableSplit, tac);
      } catch (InterruptedException e) {
        closeTable(); // Free up the HTable connections
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
        throw new IOException("Failed to initialize RecordReader", e);
      }

@@ -116,6 +128,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
        public void close() throws IOException {
          recordReader.close();
          closeTable();
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
        }

        @Override
@@ -442,7 +458,12 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
      }

      String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME);
- setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName)));
+ if (conn == null) {
+ conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf));
+ }
+ TableName tableName = TableName.valueOf(hbaseTableName);
+ initializeTable(conn, tableName);
+
      String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING);
      boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true);

@@ -509,6 +530,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
        return results;
      } finally {
        closeTable();
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
      }
    }

@@ -516,6 +541,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
    protected void finalize() throws Throwable {
      try {
        closeTable();
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
      } finally {
        super.finalize();
      }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 24, '16 at 12:50a
activeApr 24, '16 at 12:50a
posts1
users1
websitehive.apache.org

1 user in discussion

Ctang: 1 post

People

Translate

site design / logo © 2021 Grokbase