Grokbase Groups Hive commits May 2016
FAQ
HIVE-13622 WriteSet tracking optimizations (Eugene Koifman, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0b532fc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0b532fc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0b532fc

Branch: refs/heads/branch-1
Commit: c0b532fceec3ae06f06eb94c0075ac7e86187c35
Parents: 6c65832
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Thu May 19 12:50:06 2016 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Thu May 19 12:50:06 2016 -0700

----------------------------------------------------------------------
  .../hive/hcatalog/streaming/HiveEndPoint.java | 4 +-
  .../hive/metastore/TestHiveMetaStoreTxns.java | 7 +
  metastore/if/hive_metastore.thrift | 12 +
  .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 1814 +++++++++---------
  .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 905 +++++----
  .../gen/thrift/gen-cpp/hive_metastore_types.h | 52 +-
  .../metastore/api/AddDynamicPartitions.java | 127 +-
  .../hive/metastore/api/DataOperationType.java | 57 +
  .../hive/metastore/api/LockComponent.java | 234 ++-
  .../src/gen/thrift/gen-php/metastore/Types.php | 86 +
  .../gen/thrift/gen-py/hive_metastore/ttypes.py | 69 +-
  .../gen/thrift/gen-rb/hive_metastore_types.rb | 27 +-
  .../hive/metastore/HiveMetaStoreClient.java | 9 +
  .../hadoop/hive/metastore/IMetaStoreClient.java | 11 +-
  .../hive/metastore/LockComponentBuilder.java | 10 +
  .../hive/metastore/LockRequestBuilder.java | 9 +
  .../hadoop/hive/metastore/txn/TxnHandler.java | 108 +-
  .../metastore/txn/TestCompactionTxnHandler.java | 16 +-
  .../hive/metastore/txn/TestTxnHandler.java | 84 +
  .../apache/hadoop/hive/ql/exec/MoveTask.java | 3 +-
  .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 18 +-
  .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 34 +-
  .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +-
  .../hive/ql/lockmgr/TestDbTxnManager2.java | 181 +-
  .../hive/ql/txn/compactor/TestCleaner.java | 6 +
  .../hive/ql/txn/compactor/TestInitiator.java | 20 +
  .../gen-py/hive_service/ThriftHive-remote | 7 +
  27 files changed, 2473 insertions(+), 1443 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 41d2834..b9c1e23 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
  import org.apache.hadoop.hive.metastore.LockComponentBuilder;
  import org.apache.hadoop.hive.metastore.LockRequestBuilder;
  import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
  import org.apache.hadoop.hive.metastore.api.FieldSchema;
  import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
  import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -964,7 +965,8 @@ public class HiveEndPoint {
        LockComponentBuilder lockCompBuilder = new LockComponentBuilder()
                .setDbName(hiveEndPoint.database)
                .setTableName(hiveEndPoint.table)
- .setShared();
+ .setShared()
+ .setOperationType(DataOperationType.INSERT);
        if (partNameForLock!=null && !partNameForLock.isEmpty() ) {
            lockCompBuilder.setPartitionName(partNameForLock);
        }

http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
index d5ecf98..f0646f6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
@@ -21,6 +21,7 @@ import junit.framework.Assert;
  import org.apache.hadoop.hive.common.ValidTxnList;
  import org.apache.hadoop.hive.common.ValidReadTxnList;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
  import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
  import org.apache.hadoop.hive.metastore.api.LockResponse;
  import org.apache.hadoop.hive.metastore.api.LockState;
@@ -151,14 +152,17 @@ public class TestHiveMetaStoreTxns {
          .setTableName("mytable")
          .setPartitionName("mypartition")
          .setExclusive()
+ .setOperationType(DataOperationType.NO_TXN)
          .build());
      rqstBuilder.addLockComponent(new LockComponentBuilder()
          .setDbName("mydb")
          .setTableName("yourtable")
          .setSemiShared()
+ .setOperationType(DataOperationType.NO_TXN)
          .build());
      rqstBuilder.addLockComponent(new LockComponentBuilder()
          .setDbName("yourdb")
+ .setOperationType(DataOperationType.NO_TXN)
          .setShared()
          .build());
      rqstBuilder.setUser("fred");
@@ -187,15 +191,18 @@ public class TestHiveMetaStoreTxns {
          .setTableName("mytable")
          .setPartitionName("mypartition")
          .setSemiShared()
+ .setOperationType(DataOperationType.UPDATE)
          .build())
        .addLockComponent(new LockComponentBuilder()
          .setDbName("mydb")
          .setTableName("yourtable")
          .setSemiShared()
+ .setOperationType(DataOperationType.UPDATE)
          .build())
        .addLockComponent(new LockComponentBuilder()
          .setDbName("yourdb")
          .setShared()
+ .setOperationType(DataOperationType.SELECT)
          .build())
        .setUser("fred");


http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
index 4b5d207..f3d1d42 100755
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@ -106,6 +106,15 @@ enum GrantRevokeType {
      REVOKE = 2,
  }

+enum DataOperationType {
+ SELECT = 1,
+ INSERT = 2
+ UPDATE = 3,
+ DELETE = 4,
+ UNSET = 5,//this is the default to distinguish from NULL from old clients
+ NO_TXN = 6,//drop table, insert overwrite, etc - something non-transactional
+}
+
  // Types of events the client can request that the metastore fire. For now just support DML operations, as the metastore knows
  // about DDL operations and there's no reason for the client to request such an event.
  enum EventRequestType {
@@ -587,6 +596,8 @@ struct LockComponent {
      3: required string dbname,
      4: optional string tablename,
      5: optional string partitionname,
+ 6: optional DataOperationType operationType = DataOperationType.UNSET,
+ 7: optional bool isAcid = false
  }

  struct LockRequest {
@@ -692,6 +703,7 @@ struct AddDynamicPartitions {
      2: required string dbname,
      3: required string tablename,
      4: required list<string> partitionnames,
+ 5: optional DataOperationType operationType = DataOperationType.UNSET
  }

  struct NotificationEventRequest {

Search Discussions

Discussion Posts

Previous

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 10 of 11 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 19, '16 at 7:29p
activeMay 19, '16 at 8:00p
posts11
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 11 posts

People

Translate

site design / logo © 2021 Grokbase