FAQ
Author: hashutosh
Date: Sat Nov 15 21:42:25 2014
New Revision: 1639928

URL: http://svn.apache.org/r1639928
Log:
HIVE-8308 : Acid related table properties should be defined in one place and should be case insensitive (Alan Gates via Ashutosh Chauhan)

Modified:
     hive/trunk/metastore/if/hive_metastore.thrift
     hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
     hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
     hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
     hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
     hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
     hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java

Modified: hive/trunk/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/if/hive_metastore.thrift?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hive/trunk/metastore/if/hive_metastore.thrift Sat Nov 15 21:42:25 2014
@@ -1139,6 +1139,7 @@ const string META_TABLE_PARTITION_COLUMN
  const string FILE_INPUT_FORMAT = "file.inputformat",
  const string FILE_OUTPUT_FORMAT = "file.outputformat",
  const string META_TABLE_STORAGE = "storage_handler",
-
+const string TABLE_IS_TRANSACTIONAL = "transactional",
+const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",



Modified: hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp Sat Nov 15 21:42:25 2014
@@ -53,6 +53,10 @@ hive_metastoreConstants::hive_metastoreC

    META_TABLE_STORAGE = "storage_handler";

+ TABLE_IS_TRANSACTIONAL = "transactional";
+
+ TABLE_NO_AUTO_COMPACT = "no_auto_compaction";
+
  }

  }}} // namespace

Modified: hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h Sat Nov 15 21:42:25 2014
@@ -36,6 +36,8 @@ class hive_metastoreConstants {
    std::string FILE_INPUT_FORMAT;
    std::string FILE_OUTPUT_FORMAT;
    std::string META_TABLE_STORAGE;
+ std::string TABLE_IS_TRANSACTIONAL;
+ std::string TABLE_NO_AUTO_COMPACT;
  };

  extern const hive_metastoreConstants g_hive_metastore_constants;

Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java Sat Nov 15 21:42:25 2014
@@ -75,4 +75,8 @@ public class hive_metastoreConstants {

    public static final String META_TABLE_STORAGE = "storage_handler";

+ public static final String TABLE_IS_TRANSACTIONAL = "transactional";
+
+ public static final String TABLE_NO_AUTO_COMPACT = "no_auto_compaction";
+
  }

Modified: hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php Sat Nov 15 21:42:25 2014
@@ -12890,4 +12890,8 @@ $GLOBALS['hive_metastore_CONSTANTS']['FI

  $GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_STORAGE'] = "storage_handler";

+$GLOBALS['hive_metastore_CONSTANTS']['TABLE_IS_TRANSACTIONAL'] = "transactional";
+
+$GLOBALS['hive_metastore_CONSTANTS']['TABLE_NO_AUTO_COMPACT'] = "no_auto_compaction";
+


Modified: hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py Sat Nov 15 21:42:25 2014
@@ -30,3 +30,5 @@ META_TABLE_PARTITION_COLUMN_TYPES = "par
  FILE_INPUT_FORMAT = "file.inputformat"
  FILE_OUTPUT_FORMAT = "file.outputformat"
  META_TABLE_STORAGE = "storage_handler"
+TABLE_IS_TRANSACTIONAL = "transactional"
+TABLE_NO_AUTO_COMPACT = "no_auto_compaction"

Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb Sat Nov 15 21:42:25 2014
@@ -49,3 +49,7 @@ FILE_OUTPUT_FORMAT = %q"file.outputforma

  META_TABLE_STORAGE = %q"storage_handler"

+TABLE_IS_TRANSACTIONAL = %q"transactional"
+
+TABLE_NO_AUTO_COMPACT = %q"no_auto_compaction"
+

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Nov 15 21:42:25 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.FieldSchema;
  import org.apache.hadoop.hive.metastore.api.MetaException;
  import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
  import org.apache.hadoop.hive.ql.ErrorMsg;
  import org.apache.hadoop.hive.ql.QueryProperties;
  import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
@@ -318,9 +319,6 @@ public class SemanticAnalyzer extends Ba

    private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";

- @VisibleForTesting
- static final String ACID_TABLE_PROPERTY = "transactional";
-
    private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
    private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
    private HashMap<String, Operator<? extends OperatorDesc>> topOps;
@@ -12415,7 +12413,9 @@ public class SemanticAnalyzer extends Ba
    private boolean isAcidTable(Table tab) {
      if (tab == null) return false;
      if (!SessionState.get().getTxnMgr().supportsAcid()) return false;
- return tab.getProperty(ACID_TABLE_PROPERTY) != null;
+ String tableIsTransactional =
+ tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
    }

    private boolean isAcidOutputFormat(Class<? extends HiveOutputFormat> of) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Sat Nov 15 21:42:25 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
  import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
  import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
  import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
  import org.apache.hadoop.hive.metastore.txn.TxnHandler;
  import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -53,8 +54,6 @@ public class Initiator extends Compactor
    static final private String CLASS_NAME = Initiator.class.getName();
    static final private Log LOG = LogFactory.getLog(CLASS_NAME);

- static final private String NO_COMPACTION = "NO_AUTO_COMPACTION";
-
    private long checkInterval;

    @Override
@@ -86,9 +85,8 @@ public class Initiator extends Compactor
              try {
                Table t = resolveTable(ci);
                // check if no compaction set for this table
- if (t.getParameters().get(NO_COMPACTION) != null) {
- LOG.info("Table " + tableName(t) + " marked " + NO_COMPACTION +
- " so we will not compact it.");
+ if (noAutoCompactSet(t)) {
+ LOG.info("Table " + tableName(t) + " marked true so we will not compact it.");
                  continue;
                }

@@ -279,4 +277,16 @@ public class Initiator extends Compactor
      rqst.setRunas(runAs);
      txnHandler.compact(rqst);
    }
+
+ // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then was moved
+ // to no_auto_compact, we need to check it in both cases.
+ private boolean noAutoCompactSet(Table t) {
+ String noAutoCompact =
+ t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT);
+ if (noAutoCompact == null) {
+ noAutoCompact =
+ t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase());
+ }
+ return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true");
+ }
  }

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java Sat Nov 15 21:42:25 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputS
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
  import org.apache.hadoop.hive.ql.Context;
  import org.apache.hadoop.hive.ql.QueryPlan;
  import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -267,7 +268,7 @@ public class TestUpdateDeleteSemanticAna
      // I have to create the tables here (rather than in setup()) because I need the Hive
      // connection, which is conviently created by the semantic analyzer.
      Map<String, String> params = new HashMap<String, String>(1);
- params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true");
+ params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
      db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class,
          OrcOutputFormat.class, 2, Arrays.asList("a"), params);
      db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class,

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java Sat Nov 15 21:42:25 2014
@@ -243,6 +243,32 @@ public class TestInitiator extends Compa
    }

    @Test
+ public void noCompactWhenNoCompactSetLowerCase() throws Exception {
+ Map<String, String> parameters = new HashMap<String, String>(1);
+ parameters.put("no_auto_compaction", "true");
+ Table t = newTable("default", "ncwncs", false, parameters);
+
+ HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
+
+ for (int i = 0; i < 11; i++) {
+ long txnid = openTxn();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
+ comp.setTablename("ncwncs");
+ List<LockComponent> components = new ArrayList<LockComponent>(1);
+ components.add(comp);
+ LockRequest req = new LockRequest(components, "me", "localhost");
+ req.setTxnid(txnid);
+ LockResponse res = txnHandler.lock(req);
+ txnHandler.abortTxn(new AbortTxnRequest(txnid));
+ }
+
+ startInitiator();
+
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ Assert.assertEquals(0, rsp.getCompactsSize());
+ }
+
+ @Test
    public void noCompactWhenCompactAlreadyScheduled() throws Exception {
      Table t = newTable("default", "ncwcas", false);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 15, '14 at 9:43p
activeNov 15, '14 at 9:43p
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase