Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Sat Apr 18 19:04:46 2015
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.parse;
import java.io.IOException;
+import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -36,9 +37,11 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -56,6 +59,7 @@ import org.apache.hadoop.hive.ql.plan.Ad
import org.apache.hadoop.hive.ql.plan.CopyWork;
import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.DropTableDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -84,133 +88,124 @@ public class ImportSemanticAnalyzer exte
public void analyzeInternal(ASTNode ast) throws SemanticException {
try {
Tree fromTree = ast.getChild(0);
- // initialize load path
- String tmpPath = stripQuotes(fromTree.getText());
- URI fromURI = EximUtil.getValidatedURI(conf, tmpPath);
+ boolean isLocationSet = false;
+ boolean isExternalSet = false;
+ boolean isTableSet = false;
+ boolean isDbNameSet = false;
+ boolean isPartSpecSet = false;
+ String parsedLocation = null;
+ String parsedTableName = null;
+ String parsedDbName = null;
+ LinkedHashMap<String, String> parsedPartSpec = new LinkedHashMap<String, String>();
+
+ for (int i = 1; i < ast.getChildCount(); ++i){
+ ASTNode child = (ASTNode) ast.getChild(i);
+ switch (child.getToken().getType()){
+ case HiveParser.KW_EXTERNAL:
+ isExternalSet = true;
+ break;
+ case HiveParser.TOK_TABLELOCATION:
+ isLocationSet = true;
+ parsedLocation = EximUtil.relativeToAbsolutePath(conf, unescapeSQLString(child.getChild(0).getText()));
+ break;
+ case HiveParser.TOK_TAB:
+ isTableSet = true;
+ ASTNode tableNameNode = (ASTNode) child.getChild(0);
+ Map.Entry<String,String> dbTablePair = getDbTableNamePair(tableNameNode);
+ parsedDbName = dbTablePair.getKey();
+ parsedTableName = dbTablePair.getValue();
+ if (parsedDbName != null){
+ isDbNameSet = true;
+ }
+ // get partition metadata if partition specified
+ if (child.getChildCount() == 2) {
+ ASTNode partspec = (ASTNode) child.getChild(1);
+ isPartSpecSet = true;
+ parsePartitionSpec(child, parsedPartSpec);
+ }
+ break;
+ }
+ }
+
+ // parsing statement is now done, on to logic.
+
+ // initialize load path
+ URI fromURI = EximUtil.getValidatedURI(conf, stripQuotes(fromTree.getText()));
FileSystem fs = FileSystem.get(fromURI, conf);
- String dbname = null;
- CreateTableDesc tblDesc = null;
- List<AddPartitionDesc> partitionDescs = new ArrayList<AddPartitionDesc>();
- Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(),
- fromURI.getPath());
+ Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath());
inputs.add(toReadEntity(fromPath));
+
+ EximUtil.ReadMetaData rv = new EximUtil.ReadMetaData();
try {
- Path metadataPath = new Path(fromPath, METADATA_NAME);
- Map.Entry<org.apache.hadoop.hive.metastore.api.Table,
- List<Partition>> rv = EximUtil.readMetaData(fs, metadataPath);
- dbname = SessionState.get().getCurrentDatabase();
- org.apache.hadoop.hive.metastore.api.Table table = rv.getKey();
- tblDesc = new CreateTableDesc(
- table.getTableName(),
- false, // isExternal: set to false here, can be overwritten by the
- // IMPORT stmt
- table.isTemporary(),
- table.getSd().getCols(),
- table.getPartitionKeys(),
- table.getSd().getBucketCols(),
- table.getSd().getSortCols(),
- table.getSd().getNumBuckets(),
- null, null, null, null, null, // these 5 delims passed as serde params
- null, // comment passed as table params
- table.getSd().getInputFormat(),
- table.getSd().getOutputFormat(),
- null, // location: set to null here, can be
- // overwritten by the IMPORT stmt
- table.getSd().getSerdeInfo().getSerializationLib(),
- null, // storagehandler passed as table params
- table.getSd().getSerdeInfo().getParameters(),
- table.getParameters(), false,
- (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo()
- .getSkewedColNames(),
- (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo()
- .getSkewedColValues());
- tblDesc.setStoredAsSubDirectories(table.getSd().isStoredAsSubDirectories());
-
- List<FieldSchema> partCols = tblDesc.getPartCols();
- List<String> partColNames = new ArrayList<String>(partCols.size());
- for (FieldSchema fsc : partCols) {
- partColNames.add(fsc.getName());
- }
- List<Partition> partitions = rv.getValue();
- for (Partition partition : partitions) {
- // TODO: this should not create AddPartitionDesc per partition
- AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
- EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()),
- partition.getSd().getLocation(), partition.getParameters());
- AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0);
- partDesc.setInputFormat(partition.getSd().getInputFormat());
- partDesc.setOutputFormat(partition.getSd().getOutputFormat());
- partDesc.setNumBuckets(partition.getSd().getNumBuckets());
- partDesc.setCols(partition.getSd().getCols());
- partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib());
- partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters());
- partDesc.setBucketCols(partition.getSd().getBucketCols());
- partDesc.setSortCols(partition.getSd().getSortCols());
- partDesc.setLocation(new Path(fromPath,
- Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString());
- partitionDescs.add(partsDesc);
- }
+ rv = EximUtil.readMetaData(fs, new Path(fromPath, METADATA_NAME));
} catch (IOException e) {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
}
- LOG.debug("metadata read and parsed");
- for (int i = 1; i < ast.getChildCount(); ++i) {
- ASTNode child = (ASTNode) ast.getChild(i);
- switch (child.getToken().getType()) {
- case HiveParser.KW_EXTERNAL:
- tblDesc.setExternal(true);
- break;
- case HiveParser.TOK_TABLELOCATION:
- String location = unescapeSQLString(child.getChild(0).getText());
- location = EximUtil.relativeToAbsolutePath(conf, location);
- inputs.add(toReadEntity(location));
- tblDesc.setLocation(location);
- break;
- case HiveParser.TOK_TAB:
- Tree tableTree = child.getChild(0);
- // initialize destination table/partition
- String tableName = getUnescapedName((ASTNode)tableTree);
- tblDesc.setTableName(tableName);
- // get partition metadata if partition specified
- LinkedHashMap<String, String> partSpec = new LinkedHashMap<String, String>();
- if (child.getChildCount() == 2) {
- ASTNode partspec = (ASTNode) child.getChild(1);
- // partSpec is a mapping from partition column name to its value.
- for (int j = 0; j < partspec.getChildCount(); ++j) {
- ASTNode partspec_val = (ASTNode) partspec.getChild(j);
- String val = null;
- String colName = unescapeIdentifier(partspec_val.getChild(0)
- .getText().toLowerCase());
- if (partspec_val.getChildCount() < 2) { // DP in the form of T
- // partition (ds, hr)
- throw new SemanticException(
- ErrorMsg.INVALID_PARTITION
- .getMsg(" - Dynamic partitions not allowed"));
- } else { // in the form of T partition (ds="2010-03-03")
- val = stripQuotes(partspec_val.getChild(1).getText());
- }
- partSpec.put(colName, val);
- }
- boolean found = false;
- for (Iterator<AddPartitionDesc> partnIter = partitionDescs
- .listIterator(); partnIter.hasNext();) {
- AddPartitionDesc addPartitionDesc = partnIter.next();
- if (!found && addPartitionDesc.getPartition(0).getPartSpec().equals(partSpec)) {
- found = true;
- } else {
- partnIter.remove();
- }
- }
- if (!found) {
- throw new SemanticException(
- ErrorMsg.INVALID_PARTITION
- .getMsg(" - Specified partition not found in import directory"));
- }
+
+ ReplicationSpec replicationSpec = rv.getReplicationSpec();
+ if (replicationSpec.isNoop()){
+ // nothing to do here, silently return.
+ return;
+ }
+
+ String dbname = SessionState.get().getCurrentDatabase();
+ if (isDbNameSet){
+ // If the parsed statement contained a db.tablename specification, prefer that.
+ dbname = parsedDbName;
+ }
+
+ // Create table associated with the import
+ // Executed if relevant, and used to contain all the other details about the table if not.
+ CreateTableDesc tblDesc = getBaseCreateTableDescFromTable(dbname,rv.getTable());
+
+ if (isExternalSet){
+ tblDesc.setExternal(isExternalSet);
+ // This condition-check could have been avoided, but to honour the old
+ // default of not calling if it wasn't set, we retain that behaviour.
+ // TODO:cleanup after verification that the outer if isn't really needed here
+ }
+
+ if (isLocationSet){
+ tblDesc.setLocation(parsedLocation);
+ inputs.add(toReadEntity(parsedLocation));
+ }
+
+ if (isTableSet){
+ tblDesc.setTableName(parsedTableName);
+ }
+
+ List<AddPartitionDesc> partitionDescs = new ArrayList<AddPartitionDesc>();
+ Iterable<Partition> partitions = rv.getPartitions();
+ for (Partition partition : partitions) {
+ // TODO: this should ideally not create AddPartitionDesc per partition
+ AddPartitionDesc partsDesc = getBaseAddPartitionDescFromPartition(fromPath, dbname, tblDesc, partition);
+ partitionDescs.add(partsDesc);
+ }
+
+ if (isPartSpecSet){
+ // The import specification asked for only a particular partition to be loaded
+ // We load only that, and ignore all the others.
+ boolean found = false;
+ for (Iterator<AddPartitionDesc> partnIter = partitionDescs
+ .listIterator(); partnIter.hasNext();) {
+ AddPartitionDesc addPartitionDesc = partnIter.next();
+ if (!found && addPartitionDesc.getPartition(0).getPartSpec().equals(parsedPartSpec)) {
+ found = true;
+ } else {
+ partnIter.remove();
}
}
+ if (!found) {
+ throw new SemanticException(
+ ErrorMsg.INVALID_PARTITION
+ .getMsg(" - Specified partition not found in import directory"));
+ }
}
+
if (tblDesc.getTableName() == null) {
+ // Either we got the tablename from the IMPORT statement (first priority)
+ // or from the export dump.
throw new SemanticException(ErrorMsg.NEED_TABLE_SPECIFICATION.getMsg());
} else {
conf.set("import.destination.table", tblDesc.getTableName());
@@ -218,68 +213,26 @@ public class ImportSemanticAnalyzer exte
addPartitionDesc.setTableName(tblDesc.getTableName());
}
}
+
Warehouse wh = new Warehouse(conf);
- try {
- Table table = db.getTable(tblDesc.getTableName());
- checkTable(table, tblDesc);
- LOG.debug("table " + tblDesc.getTableName()
- + " exists: metadata checked");
+ Table table = tableIfExists(tblDesc);
+
+ if (table != null){
+ checkTable(table, tblDesc,replicationSpec);
+ LOG.debug("table " + tblDesc.getTableName() + " exists: metadata checked");
tableExists = true;
- conf.set("import.destination.dir", table.getDataLocation().toString());
- if (table.isPartitioned()) {
- LOG.debug("table partitioned");
- for (AddPartitionDesc addPartitionDesc : partitionDescs) {
- Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
- if (db.getPartition(table, partSpec, false) == null) {
- rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc));
- } else {
- throw new SemanticException(
- ErrorMsg.PARTITION_EXISTS.getMsg(partSpecToString(partSpec)));
- }
- }
- } else {
- LOG.debug("table non-partitioned");
- checkTargetLocationEmpty(fs, new Path(table.getDataLocation()
- .toString()));
- loadTable(fromURI, table);
- }
- // Set this to read because we can't overwrite any existing partitions
- outputs.add(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK));
- } catch (InvalidTableException e) {
- LOG.debug("table " + tblDesc.getTableName() + " does not exist");
-
- Task<?> t = TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- tblDesc), conf);
- Table table = new Table(dbname, tblDesc.getTableName());
- String currentDb = SessionState.get().getCurrentDatabase();
- conf.set("import.destination.dir",
- wh.getTablePath(db.getDatabaseCurrent(),
- tblDesc.getTableName()).toString());
- if ((tblDesc.getPartCols() != null) && (tblDesc.getPartCols().size() != 0)) {
- for (AddPartitionDesc addPartitionDesc : partitionDescs) {
- t.addDependentTask(
- addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc));
- }
- } else {
- LOG.debug("adding dependent CopyWork/MoveWork for table");
- if (tblDesc.isExternal() && (tblDesc.getLocation() == null)) {
- LOG.debug("Importing in place, no emptiness check, no copying/loading");
- Path dataPath = new Path(fromURI.toString(), "data");
- tblDesc.setLocation(dataPath.toString());
- } else {
- Path tablePath = null;
- if (tblDesc.getLocation() != null) {
- tablePath = new Path(tblDesc.getLocation());
- } else {
- tablePath = wh.getTablePath(db.getDatabaseCurrent(), tblDesc.getTableName());
- }
- checkTargetLocationEmpty(fs, tablePath);
- t.addDependentTask(loadTable(fromURI, table));
- }
- }
- rootTasks.add(t);
- //inputs.add(new ReadEntity(fromURI.toString(),
- // fromURI.getScheme().equals("hdfs") ? true : false));
+ }
+
+ if (!replicationSpec.isInReplicationScope()){
+ createRegularImportTasks(
+ rootTasks, tblDesc, partitionDescs,
+ isPartSpecSet, replicationSpec, table,
+ fromURI, fs, wh);
+ } else {
+ createReplImportTasks(
+ rootTasks, tblDesc, partitionDescs,
+ isPartSpecSet, replicationSpec, table,
+ fromURI, fs, wh);
}
} catch (SemanticException e) {
throw e;
@@ -288,14 +241,90 @@ public class ImportSemanticAnalyzer exte
}
}
- private Task<?> loadTable(URI fromURI, Table table) {
+ private void parsePartitionSpec(ASTNode tableNode, LinkedHashMap<String, String> partSpec) throws SemanticException {
+ // get partition metadata if partition specified
+ if (tableNode.getChildCount() == 2) {
+ ASTNode partspec = (ASTNode) tableNode.getChild(1);
+ // partSpec is a mapping from partition column name to its value.
+ for (int j = 0; j < partspec.getChildCount(); ++j) {
+ ASTNode partspec_val = (ASTNode) partspec.getChild(j);
+ String val = null;
+ String colName = unescapeIdentifier(partspec_val.getChild(0)
+ .getText().toLowerCase());
+ if (partspec_val.getChildCount() < 2) { // DP in the form of T
+ // partition (ds, hr)
+ throw new SemanticException(
+ ErrorMsg.INVALID_PARTITION
+ .getMsg(" - Dynamic partitions not allowed"));
+ } else { // in the form of T partition (ds="2010-03-03")
+ val = stripQuotes(partspec_val.getChild(1).getText());
+ }
+ partSpec.put(colName, val);
+ }
+ }
+ }
+
+ private AddPartitionDesc getBaseAddPartitionDescFromPartition(
+ Path fromPath, String dbname, CreateTableDesc tblDesc, Partition partition) throws MetaException {
+ AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
+ EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()),
+ partition.getSd().getLocation(), partition.getParameters());
+ AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0);
+ partDesc.setInputFormat(partition.getSd().getInputFormat());
+ partDesc.setOutputFormat(partition.getSd().getOutputFormat());
+ partDesc.setNumBuckets(partition.getSd().getNumBuckets());
+ partDesc.setCols(partition.getSd().getCols());
+ partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib());
+ partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters());
+ partDesc.setBucketCols(partition.getSd().getBucketCols());
+ partDesc.setSortCols(partition.getSd().getSortCols());
+ partDesc.setLocation(new Path(fromPath,
+ Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString());
+ return partsDesc;
+ }
+
+ private CreateTableDesc getBaseCreateTableDescFromTable(String dbName,
+ org.apache.hadoop.hive.metastore.api.Table table) {
+ if ((table.getPartitionKeys() == null) || (table.getPartitionKeys().size() == 0)){
+ table.putToParameters(StatsSetupConst.DO_NOT_UPDATE_STATS,"true");
+ }
+ CreateTableDesc tblDesc = new CreateTableDesc(
+ dbName,
+ table.getTableName(),
+ false, // isExternal: set to false here, can be overwritten by the
+ // IMPORT stmt
+ table.isTemporary(),
+ table.getSd().getCols(),
+ table.getPartitionKeys(),
+ table.getSd().getBucketCols(),
+ table.getSd().getSortCols(),
+ table.getSd().getNumBuckets(),
+ null, null, null, null, null, // these 5 delims passed as serde params
+ null, // comment passed as table params
+ table.getSd().getInputFormat(),
+ table.getSd().getOutputFormat(),
+ null, // location: set to null here, can be
+ // overwritten by the IMPORT stmt
+ table.getSd().getSerdeInfo().getSerializationLib(),
+ null, // storagehandler passed as table params
+ table.getSd().getSerdeInfo().getParameters(),
+ table.getParameters(), false,
+ (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo()
+ .getSkewedColNames(),
+ (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo()
+ .getSkewedColValues());
+ tblDesc.setStoredAsSubDirectories(table.getSd().isStoredAsSubDirectories());
+ return tblDesc;
+ }
+
+ private Task<?> loadTable(URI fromURI, Table table, boolean replace) {
Path dataPath = new Path(fromURI.toString(), "data");
Path tmpPath = ctx.getExternalTmpPath(new Path(fromURI));
Task<?> copyTask = TaskFactory.get(new CopyWork(dataPath,
tmpPath, false), conf);
LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
Utilities.getTableDesc(table), new TreeMap<String, String>(),
- false);
+ replace);
Task<?> loadTableTask = TaskFactory.get(new MoveWork(getInputs(),
getOutputs(), loadTableWork, null, false), conf);
copyTask.addDependentTask(loadTableTask);
@@ -303,9 +332,49 @@ public class ImportSemanticAnalyzer exte
return loadTableTask;
}
- private Task<?> addSinglePartition(URI fromURI, FileSystem fs, CreateTableDesc tblDesc,
+ private Task<?> createTableTask(CreateTableDesc tableDesc){
+ return TaskFactory.get(new DDLWork(
+ getInputs(),
+ getOutputs(),
+ tableDesc
+ ), conf);
+ }
+
+ private Task<?> dropTableTask(Table table){
+ return TaskFactory.get(new DDLWork(
+ getInputs(),
+ getOutputs(),
+ new DropTableDesc(table.getTableName(), false, true, true, null)
+ ), conf);
+ }
+
+ private Task<? extends Serializable> alterTableTask(CreateTableDesc tableDesc) {
+ tableDesc.setReplaceMode(true);
+ return TaskFactory.get(new DDLWork(
+ getInputs(),
+ getOutputs(),
+ tableDesc
+ ), conf);
+ }
+
+ private Task<? extends Serializable> alterSinglePartition(
+ URI fromURI, FileSystem fs, CreateTableDesc tblDesc,
+ Table table, Warehouse wh, AddPartitionDesc addPartitionDesc,
+ ReplicationSpec replicationSpec, org.apache.hadoop.hive.ql.metadata.Partition ptn) {
+ addPartitionDesc.setReplaceMode(true);
+ addPartitionDesc.getPartition(0).setLocation(ptn.getLocation()); // use existing location
+ return TaskFactory.get(new DDLWork(
+ getInputs(),
+ getOutputs(),
+ addPartitionDesc
+ ), conf);
+ }
+
+
+ private Task<?> addSinglePartition(URI fromURI, FileSystem fs, CreateTableDesc tblDesc,
Table table, Warehouse wh,
- AddPartitionDesc addPartitionDesc) throws MetaException, IOException, HiveException {
+ AddPartitionDesc addPartitionDesc, ReplicationSpec replicationSpec)
+ throws MetaException, IOException, HiveException {
AddPartitionDesc.OnePartitionDesc partSpec = addPartitionDesc.getPartition(0);
if (tblDesc.isExternal() && tblDesc.getLocation() == null) {
LOG.debug("Importing in-place: adding AddPart for partition "
@@ -316,22 +385,7 @@ public class ImportSemanticAnalyzer exte
return addPartTask;
} else {
String srcLocation = partSpec.getLocation();
- Path tgtPath = null;
- if (tblDesc.getLocation() == null) {
- if (table.getDataLocation() != null) {
- tgtPath = new Path(table.getDataLocation().toString(),
- Warehouse.makePartPath(partSpec.getPartSpec()));
- } else {
- tgtPath = new Path(wh.getTablePath(
- db.getDatabaseCurrent(), tblDesc.getTableName()),
- Warehouse.makePartPath(partSpec.getPartSpec()));
- }
- } else {
- tgtPath = new Path(tblDesc.getLocation(),
- Warehouse.makePartPath(partSpec.getPartSpec()));
- }
- checkTargetLocationEmpty(fs, tgtPath);
- partSpec.setLocation(tgtPath.toString());
+ fixLocationInPartSpec(fs, tblDesc, table, wh, replicationSpec, partSpec);
LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition "
+ partSpecToString(partSpec.getPartSpec())
+ " with source location: " + srcLocation);
@@ -354,8 +408,38 @@ public class ImportSemanticAnalyzer exte
}
}
- private void checkTargetLocationEmpty(FileSystem fs, Path targetPath)
+ /**
+ * Helper method to set location properly in partSpec
+ */
+ private void fixLocationInPartSpec(
+ FileSystem fs, CreateTableDesc tblDesc, Table table,
+ Warehouse wh, ReplicationSpec replicationSpec,
+ AddPartitionDesc.OnePartitionDesc partSpec) throws MetaException, HiveException, IOException {
+ Path tgtPath = null;
+ if (tblDesc.getLocation() == null) {
+ if (table.getDataLocation() != null) {
+ tgtPath = new Path(table.getDataLocation().toString(),
+ Warehouse.makePartPath(partSpec.getPartSpec()));
+ } else {
+ Database parentDb = db.getDatabase(tblDesc.getDatabaseName());
+ tgtPath = new Path(
+ wh.getTablePath( parentDb, tblDesc.getTableName()),
+ Warehouse.makePartPath(partSpec.getPartSpec()));
+ }
+ } else {
+ tgtPath = new Path(tblDesc.getLocation(),
+ Warehouse.makePartPath(partSpec.getPartSpec()));
+ }
+ checkTargetLocationEmpty(fs, tgtPath, replicationSpec);
+ partSpec.setLocation(tgtPath.toString());
+ }
+
+ private void checkTargetLocationEmpty(FileSystem fs, Path targetPath, ReplicationSpec replicationSpec)
throws IOException, SemanticException {
+ if (replicationSpec.isInReplicationScope()){
+ // replication scope allows replacement, and does not require empty directories
+ return;
+ }
LOG.debug("checking emptiness of " + targetPath.toString());
if (fs.exists(targetPath)) {
FileStatus[] status = fs.listStatus(targetPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
@@ -382,38 +466,57 @@ public class ImportSemanticAnalyzer exte
return sb.toString();
}
- private static void checkTable(Table table, CreateTableDesc tableDesc)
+ private void checkTable(Table table, CreateTableDesc tableDesc, ReplicationSpec replicationSpec)
throws SemanticException, URISyntaxException {
+ // This method gets called only in the scope that a destination table already exists, so
+ // we're validating if the table is an appropriate destination to import into
+
+ if (replicationSpec.isInReplicationScope()){
+ // If this import is being done for replication, then this will be a managed table, and replacements
+ // are allowed irrespective of what the table currently looks like. So no more checks are necessary.
+ return;
+ } else {
+ // verify if table has been the target of replication, and if so, check HiveConf if we're allowed
+ // to override. If not, fail.
+ if (table.getParameters().containsKey(ReplicationSpec.KEY.CURR_STATE_ID.toString())
+ && conf.getBoolVar(HiveConf.ConfVars.HIVE_EXIM_RESTRICT_IMPORTS_INTO_REPLICATED_TABLES)){
+ throw new SemanticException(ErrorMsg.IMPORT_INTO_STRICT_REPL_TABLE.getMsg(
+ "Table "+table.getTableName()+" has repl.last.id parameter set." ));
+ }
+ }
+
+ // Next, we verify that the destination table is not offline, a view, or a non-native table
+ EximUtil.validateTable(table);
+
+ // If the import statement specified that we're importing to an external
+ // table, we seem to be doing the following:
+ // a) We don't allow replacement in an unpartitioned pre-existing table
+ // b) We don't allow replacement in a partitioned pre-existing table where that table is external
+ // TODO : Does this simply mean we don't allow replacement in external tables if they already exist?
+ // If so(i.e. the check is superfluous and wrong), this can be a simpler check. If not, then
+ // what we seem to be saying is that the only case we allow is to allow an IMPORT into an EXTERNAL
+ // table in the statement, if a destination partitioned table exists, so long as it is actually
+ // not external itself. Is that the case? Why?
{
- EximUtil.validateTable(table);
- if (!table.isPartitioned()) {
- if (tableDesc.isExternal()) { // the import statement specified external
- throw new SemanticException(
- ErrorMsg.INCOMPATIBLE_SCHEMA
- .getMsg(" External table cannot overwrite existing table."
- + " Drop existing table first."));
- }
- } else {
- if (tableDesc.isExternal()) { // the import statement specified external
- if (!table.getTableType().equals(TableType.EXTERNAL_TABLE)) {
- throw new SemanticException(
- ErrorMsg.INCOMPATIBLE_SCHEMA
- .getMsg(" External table cannot overwrite existing table."
- + " Drop existing table first."));
- }
- }
+ if ( (tableDesc.isExternal()) // IMPORT statement speicified EXTERNAL
+ && (!table.isPartitioned() || !table.getTableType().equals(TableType.EXTERNAL_TABLE))
+ ){
+ throw new SemanticException(ErrorMsg.INCOMPATIBLE_SCHEMA.getMsg(
+ " External table cannot overwrite existing table. Drop existing table first."));
}
}
+
+ // If a table import statement specified a location and the table(unpartitioned)
+ // already exists, ensure that the locations are the same.
+ // Partitioned tables not checked here, since the location provided would need
+ // checking against the partition in question instead.
{
- if (!table.isPartitioned()) {
- if (tableDesc.getLocation() != null) { // IMPORT statement specified
- // location
- if (!table.getDataLocation()
- .equals(new Path(tableDesc.getLocation()))) {
- throw new SemanticException(
- ErrorMsg.INCOMPATIBLE_SCHEMA.getMsg(" Location does not match"));
- }
- }
+ if ((tableDesc.getLocation() != null)
+ && (!table.isPartitioned())
+ && (!table.getDataLocation().equals(new Path(tableDesc.getLocation()))) ){
+ throw new SemanticException(
+ ErrorMsg.INCOMPATIBLE_SCHEMA.getMsg(" Location does not match"));
+
}
}
{
@@ -572,4 +675,221 @@ public class ImportSemanticAnalyzer exte
}
return null;
}
+
+ /**
+ * Create tasks for regular import, no repl complexity
+ */
+ private void createRegularImportTasks(
+ List<Task<? extends Serializable>> rootTasks,
+ CreateTableDesc tblDesc,
+ List<AddPartitionDesc> partitionDescs,
+ boolean isPartSpecSet,
+ ReplicationSpec replicationSpec,
+ Table table, URI fromURI, FileSystem fs, Warehouse wh)
+ throws HiveException, URISyntaxException, IOException, MetaException {
+
+ if (table != null){
+ if (table.isPartitioned()) {
+ LOG.debug("table partitioned");
+
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+ Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
+ org.apache.hadoop.hive.ql.metadata.Partition ptn = null;
+ if ((ptn = db.getPartition(table, partSpec, false)) == null) {
+ rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec));
+ } else {
+ throw new SemanticException(
+ ErrorMsg.PARTITION_EXISTS.getMsg(partSpecToString(partSpec)));
+ }
+ }
+
+ } else {
+ LOG.debug("table non-partitioned");
+ // ensure if destination is not empty only for regular import
+ checkTargetLocationEmpty(fs, new Path(table.getDataLocation().toString()), replicationSpec);
+ loadTable(fromURI, table, false);
+ }
+ // Set this to read because we can't overwrite any existing partitions
+ outputs.add(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK));
+ } else {
+ LOG.debug("table " + tblDesc.getTableName() + " does not exist");
+
+ Task<?> t = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), tblDesc), conf);
+ table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
+ Database parentDb = db.getDatabase(tblDesc.getDatabaseName());
+
+ if (isPartitioned(tblDesc)) {
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+ t.addDependentTask(
+ addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec));
+ }
+ } else {
+ LOG.debug("adding dependent CopyWork/MoveWork for table");
+ if (tblDesc.isExternal() && (tblDesc.getLocation() == null)) {
+ LOG.debug("Importing in place, no emptiness check, no copying/loading");
+ Path dataPath = new Path(fromURI.toString(), "data");
+ tblDesc.setLocation(dataPath.toString());
+ } else {
+ Path tablePath = null;
+ if (tblDesc.getLocation() != null) {
+ tablePath = new Path(tblDesc.getLocation());
+ } else {
+ tablePath = wh.getTablePath(parentDb, tblDesc.getTableName());
+ }
+ checkTargetLocationEmpty(fs, tablePath, replicationSpec);
+ t.addDependentTask(loadTable(fromURI, table, false));
+ }
+ }
+ rootTasks.add(t);
+ }
+ }
+
+ /**
+ * Create tasks for repl import
+ */
+ private void createReplImportTasks(
+ List<Task<? extends Serializable>> rootTasks,
+ CreateTableDesc tblDesc,
+ List<AddPartitionDesc> partitionDescs,
+ boolean isPartSpecSet, ReplicationSpec replicationSpec, Table table, URI fromURI, FileSystem fs, Warehouse wh)
+ throws HiveException, URISyntaxException, IOException, MetaException {
+
+ Task dr = null;
+ WriteEntity.WriteType lockType = WriteEntity.WriteType.DDL_NO_LOCK;
+
+ if ((table != null) && (isPartitioned(tblDesc) != table.isPartitioned())){
+ // If destination table exists, but is partitioned, and we think we're writing to an unpartitioned
+ // or if destination table exists, but is unpartitioned and we think we're writing to a partitioned
+ // table, then this can only happen because there are drops in the queue that are yet to be processed.
+ // So, we check the repl.last.id of the destination, and if it's newer, we no-op. If it's older, we
+ // drop and re-create.
+ if (replicationSpec.allowReplacementInto(table)){
+ dr = dropTableTask(table);
+ lockType = WriteEntity.WriteType.DDL_EXCLUSIVE;
+ table = null; // null it out so we go into the table re-create flow.
+ } else {
+ return; // noop out of here.
+ }
+ }
+
+ Database parentDb = db.getDatabase(tblDesc.getDatabaseName());
+ if (parentDb == null){
+ throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tblDesc.getDatabaseName()));
+ }
+ if (tblDesc.getLocation() == null) {
+ tblDesc.setLocation(wh.getTablePath(parentDb, tblDesc.getTableName()).toString());
+ }
+
+ /* Note: In the following section, Metadata-only import handling logic is
+ interleaved with regular repl-import logic. The rule of thumb being
+ followed here is that MD-only imports are essentially ALTERs. They do
+ not load data, and should not be "creating" any metadata - they should
+ be replacing instead. The only place it makes sense for a MD-only import
+ to create is in the case of a table that's been dropped and recreated,
+ or in the case of an unpartitioned table. In all other cases, it should
+ behave like a noop or a pure MD alter.
+ */
+
+ if (table == null) {
+ // Either we're dropping and re-creating, or the table didn't exist, and we're creating.
+
+ if (lockType == WriteEntity.WriteType.DDL_NO_LOCK){
+ lockType = WriteEntity.WriteType.DDL_SHARED;
+ }
+
+ Task t = createTableTask(tblDesc);
+ table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
+
+ if (!replicationSpec.isMetadataOnly()) {
+ if (isPartitioned(tblDesc)) {
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+ t.addDependentTask(
+ addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec));
+ }
+ } else {
+ LOG.debug("adding dependent CopyWork/MoveWork for table");
+ t.addDependentTask(loadTable(fromURI, table, true));
+ }
+ }
+ if (dr == null){
+ // Simply create
+ rootTasks.add(t);
+ } else {
+ // Drop and recreate
+ dr.addDependentTask(t);
+ rootTasks.add(dr);
+ }
+ } else {
+ // Table existed, and is okay to replicate into, not dropping and re-creating.
+ if (table.isPartitioned()) {
+ LOG.debug("table partitioned");
+ for (AddPartitionDesc addPartitionDesc : partitionDescs) {
+
+ Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
+ org.apache.hadoop.hive.ql.metadata.Partition ptn = null;
+
+ if ((ptn = db.getPartition(table, partSpec, false)) == null) {
+ if (!replicationSpec.isMetadataOnly()){
+ rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec));
+ }
+ } else {
+ // If replicating, then the partition already existing means we need to replace, maybe, if
+ // the destination ptn's repl.last.id is older than the replacement's.
+ if (replicationSpec.allowReplacementInto(ptn)){
+ if (!replicationSpec.isMetadataOnly()){
+ rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec));
+ } else {
+ rootTasks.add(alterSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, ptn));
+ }
+ if (lockType == WriteEntity.WriteType.DDL_NO_LOCK){
+ lockType = WriteEntity.WriteType.DDL_SHARED;
+ }
+ } else {
+ // ignore this ptn, do nothing, not an error.
+ }
+ }
+
+ }
+ if (replicationSpec.isMetadataOnly() && partitionDescs.isEmpty()){
+ // MD-ONLY table alter
+ rootTasks.add(alterTableTask(tblDesc));
+ if (lockType == WriteEntity.WriteType.DDL_NO_LOCK){
+ lockType = WriteEntity.WriteType.DDL_SHARED;
+ }
+ }
+ } else {
+ LOG.debug("table non-partitioned");
+ if (!replicationSpec.allowReplacementInto(table)){
+ return; // silently return, table is newer than our replacement.
+ }
+ if (!replicationSpec.isMetadataOnly()) {
+ loadTable(fromURI, table, true); // repl-imports are replace-into
+ } else {
+ rootTasks.add(alterTableTask(tblDesc));
+ }
+ if (lockType == WriteEntity.WriteType.DDL_NO_LOCK){
+ lockType = WriteEntity.WriteType.DDL_SHARED;
+ }
+ }
+ }
+ outputs.add(new WriteEntity(table,lockType));
+
+ }
+
+ private boolean isPartitioned(CreateTableDesc tblDesc) {
+ return !(tblDesc.getPartCols() == null || tblDesc.getPartCols().isEmpty());
+ }
+
+ /**
+ * Utility method that returns a table if one corresponding to the destination
+ * tblDesc is found. Returns null if no such table is found.
+ */
+ private Table tableIfExists(CreateTableDesc tblDesc) throws HiveException {
+ try {
+ return db.getTable(tblDesc.getDatabaseName(),tblDesc.getTableName());
+ } catch (InvalidTableException e) {
+ return null;
+ }
+ }
+
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java Sat Apr 18 19:04:46 2015
@@ -86,7 +86,7 @@ public class MetaDataExportListener exte
Path outFile = new Path(metaPath, name + ImportSemanticAnalyzer.METADATA_NAME);
try {
SessionState.getConsole().printInfo("Beginning metadata export");
- EximUtil.createExportDump(fs, outFile, mTbl, null);
+ EximUtil.createExportDump(fs, outFile, mTbl, null, null);
if (moveMetadataToTrash == true) {
wh.deleteDir(metaPath, true);
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java?rev=1674557&view=auto==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java Sat Apr 18 19:04:46 2015
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *
http://www.apache.org/licenses/LICENSE-2.0+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
+
+import javax.annotation.Nullable;
+import java.text.Collator;
+import java.util.Map;
+
+/**
+ * Statements executed to handle replication have some additional
+ * information relevant to the replication subsystem - this class
+ * captures those bits of information.
+ *
+ * Typically, this corresponds to the replicationClause definition
+ * in the parser.
+ */
+public class ReplicationSpec {
+
+ private boolean isInReplicationScope = false; // default is that it's not in a repl scope
+ private boolean isMetadataOnly = false; // default is full export/import, not metadata-only
+ private String eventId = null;
+ private String currStateId = null;
+ private boolean isNoop = false;
+
+
+ // Key definitions related to replication
+ public enum KEY {
+ REPL_SCOPE("repl.scope"),
+ EVENT_ID("repl.event.id"),
+ CURR_STATE_ID("repl.last.id"),
+ NOOP("repl.noop");
+
+ private final String keyName;
+
+ KEY(String s) {
+ this.keyName = s;
+ }
+
+ @Override
+ public String toString(){
+ return keyName;
+ }
+ }
+
+ public enum SCOPE { NO_REPL, MD_ONLY, REPL };
+
+ static private Collator collator = Collator.getInstance();
+
+ /**
+ * Constructor to construct spec based on either the ASTNode that
+ * corresponds to the replication clause itself, or corresponds to
+ * the parent node, and will scan through the children to instantiate
+ * itself.
+ * @param node replicationClause node, or parent of replicationClause node
+ */
+ public ReplicationSpec(ASTNode node){
+ if (node != null){
+ if (isApplicable(node)){
+ init(node);
+ return;
+ } else {
+ for (int i = 1; i < node.getChildCount(); ++i) {
+ ASTNode child = (ASTNode) node.getChild(i);
+ if (isApplicable(child)) {
+ init(child);
+ return;
+ }
+ }
+ }
+ }
+ // If we reached here, we did not find a replication
+ // spec in the node or its immediate children. Defaults
+ // are to pretend replication is not happening, and the
+ // statement above is running as-is.
+ }
+
+ /**
+ * Default ctor that is useful for determining default states
+ */
+ public ReplicationSpec(){
+ this((ASTNode)null);
+ }
+
+ public ReplicationSpec(
+ boolean isInReplicationScope, boolean isMetadataOnly, String eventReplicationState,
+ String currentReplicationState, boolean isNoop){
+ this.isInReplicationScope = isInReplicationScope;
+ this.isMetadataOnly = isMetadataOnly;
+ this.eventId = eventReplicationState;
+ this.currStateId = currentReplicationState;
+ this.isNoop = isNoop;
+ }
+
+ public ReplicationSpec(Function<String, String> keyFetcher) {
+ String scope = keyFetcher.apply(ReplicationSpec.KEY.REPL_SCOPE.toString());
+ this.isMetadataOnly = false;
+ this.isInReplicationScope = false;
+ if (scope != null){
+ if (scope.equalsIgnoreCase("metadata")){
+ this.isMetadataOnly = true;
+ this.isInReplicationScope = true;
+ } else if (scope.equalsIgnoreCase("all")){
+ this.isInReplicationScope = true;
+ }
+ }
+ this.eventId = keyFetcher.apply(ReplicationSpec.KEY.EVENT_ID.toString());
+ this.currStateId = keyFetcher.apply(ReplicationSpec.KEY.CURR_STATE_ID.toString());
+ this.isNoop = Boolean.valueOf(keyFetcher.apply(ReplicationSpec.KEY.NOOP.toString())).booleanValue();
+ }
+
+ /**
+ * Tests if an ASTNode is a Replication Specification
+ */
+ public static boolean isApplicable(ASTNode node){
+ return (node.getToken().getType() == HiveParser.TOK_REPLICATION);
+ }
+
+ /**
+ * @param currReplState Current object state
+ * @param replacementReplState Replacement-candidate state
+ * @return whether or not a provided replacement candidate is newer(or equal) to the existing object state or not
+ */
+ public static boolean allowReplacement(String currReplState, String replacementReplState){
+ if ((currReplState == null) || (currReplState.isEmpty())) {
+ // if we have no replication state on record for the obj, allow replacement.
+ return true;
+ }
+ if ((replacementReplState == null) || (replacementReplState.isEmpty())) {
+ // if we reached this condition, we had replication state on record for the
+ // object, but its replacement has no state. Disallow replacement
+ return false;
+ }
+
+ // First try to extract a long value from the strings, and compare them.
+ // If oldReplState is less-than or equal to newReplState, allow.
+ long currReplStateLong = Long.parseLong(currReplState.replaceAll("\\D",""));
+ long replacementReplStateLong = Long.parseLong(replacementReplState.replaceAll("\\D",""));
+
+ if ((currReplStateLong != 0) || (replacementReplStateLong != 0)){
+ return ((currReplStateLong - replacementReplStateLong) <= 0);
+ }
+
+ // If the long value of both is 0, though, fall back to lexical comparison.
+
+ // Lexical comparison according to locale will suffice for now, future might add more logic
+ return (collator.compare(currReplState.toLowerCase(), replacementReplState.toLowerCase()) <= 0);
+ }
+
+ /**
+ * Determines if a current replication object(current state of dump) is allowed to
+ * replicate-replace-into a given partition
+ */
+ public boolean allowReplacementInto(Partition ptn){
+ return allowReplacement(getLastReplicatedStateFromParameters(ptn.getParameters()),this.getCurrentReplicationState());
+ }
+
+ /**
+ * Determines if a current replication event specification is allowed to
+ * replicate-replace-into a given partition
+ */
+ public boolean allowEventReplacementInto(Partition ptn){
+ return allowReplacement(getLastReplicatedStateFromParameters(ptn.getParameters()),this.getReplicationState());
+ }
+
+ /**
+ * Determines if a current replication object(current state of dump) is allowed to
+ * replicate-replace-into a given table
+ */
+ public boolean allowReplacementInto(Table table) {
+ return allowReplacement(getLastReplicatedStateFromParameters(table.getParameters()),this.getCurrentReplicationState());
+ }
+
+ /**
+ * Determines if a current replication event specification is allowed to
+ * replicate-replace-into a given table
+ */
+ public boolean allowEventReplacementInto(Table table) {
+ return allowReplacement(getLastReplicatedStateFromParameters(table.getParameters()),this.getReplicationState());
+ }
+
+ /**
+ * Returns a predicate filter to filter an Iterable<Partition> to return all partitions
+ * that the current replication event specification is allowed to replicate-replace-into
+ */
+ public Predicate<Partition> allowEventReplacementInto() {
+ return new Predicate<Partition>() {
+ @Override
+ public boolean apply(@Nullable Partition partition) {
+ if (partition == null){
+ return false;
+ }
+ return (allowEventReplacementInto(partition));
+ }
+ };
+ }
+
+ private static String getLastReplicatedStateFromParameters(Map<String, String> m) {
+ if ((m != null) && (m.containsKey(KEY.CURR_STATE_ID.toString()))){
+ return m.get(KEY.CURR_STATE_ID.toString());
+ }
+ return null;
+ }
+
+ private void init(ASTNode node){
+ // -> ^(TOK_REPLICATION $replId $isMetadataOnly)
+ isInReplicationScope = true;
+ eventId = PlanUtils.stripQuotes(node.getChild(0).getText());
+ if (node.getChildCount() > 1){
+ if (node.getChild(1).getText().toLowerCase().equals("metadata")) {
+ isMetadataOnly= true;
+ }
+ }
+ }
+
+ /**
+ * @return true if this statement is being run for the purposes of replication
+ */
+ public boolean isInReplicationScope(){
+ return isInReplicationScope;
+ }
+
+ /**
+ * @return true if this statement refers to metadata-only operation.
+ */
+ public boolean isMetadataOnly(){
+ return isMetadataOnly;
+ }
+
+ /**
+ * @return the replication state of the event that spawned this statement
+ */
+ public String getReplicationState() {
+ return eventId;
+ }
+
+ /**
+ * @return the current replication state of the wh
+ */
+ public String getCurrentReplicationState() {
+ return currStateId;
+ }
+
+ public void setCurrentReplicationState(String currStateId) {
+ this.currStateId = currStateId;
+ }
+
+ /**
+ * @return whether or not the current replication action should be a noop
+ */
+ public boolean isNoop() {
+ return isNoop;
+ }
+
+ /**
+ * @param isNoop whether or not the current replication action should be a noop
+ */
+ public void setNoop(boolean isNoop) {
+ this.isNoop = isNoop;
+ }
+
+ public String get(KEY key) {
+ switch (key){
+ case REPL_SCOPE:
+ switch (getScope()){
+ case MD_ONLY:
+ return "metadata";
+ case REPL:
+ return "all";
+ case NO_REPL:
+ return "none";
+ }
+ case EVENT_ID:
+ return getReplicationState();
+ case CURR_STATE_ID:
+ return getCurrentReplicationState();
+ case NOOP:
+ return String.valueOf(isNoop());
+ }
+ return null;
+ }
+
+ public SCOPE getScope(){
+ if (isInReplicationScope()){
+ if (isMetadataOnly()){
+ return SCOPE.MD_ONLY;
+ } else {
+ return SCOPE.REPL;
+ }
+ } else {
+ return SCOPE.NO_REPL;
+ }
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AddPartitionDesc.java Sat Apr 18 19:04:46 2015
@@ -151,6 +151,7 @@ public class AddPartitionDesc extends DD
String dbName;
boolean ifNotExists;
List<OnePartitionDesc> partitions = null;
+ boolean replaceMode = false;
/**
@@ -287,4 +288,18 @@ public class AddPartitionDesc extends DD
public OnePartitionDesc getPartition(int i) {
return this.partitions.get(i);
}
+
+ /**
+ * @param replaceMode Determine if this AddPartition should behave like a replace-into alter instead
+ */
+ public void setReplaceMode(boolean replaceMode){
+ this.replaceMode = replaceMode;
+ }
+
+ /**
+ * @return true if this AddPartition should behave like a replace-into alter instead
+ */
+ public boolean getReplaceMode() {
+ return this.replaceMode;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java Sat Apr 18 19:04:46 2015
@@ -78,6 +78,7 @@ public class CreateTableDesc extends DDL
List<List<String>> skewedColValues;
boolean isStoredAsSubDirectories = false;
boolean isTemporary = false;
+ private boolean replaceMode = false;
public CreateTableDesc() {
}
@@ -551,4 +552,17 @@ public class CreateTableDesc extends DDL
this.isTemporary = isTemporary;
}
+ /**
+ * @param replaceMode Determine if this CreateTable should behave like a replace-into alter instead
+ */
+ public void setReplaceMode(boolean replaceMode) {
+ this.replaceMode = replaceMode;
+ }
+
+ /**
+ * @return true if this CreateTable should behave like a replace-into alter instead
+ */
+ public boolean getReplaceMode() {
+ return replaceMode;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java Sat Apr 18 19:04:46 2015
@@ -22,9 +22,9 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
-
/**
* DropTableDesc.
* TODO: this is currently used for both drop table and drop partitions.
@@ -56,6 +56,7 @@ public class DropTableDesc extends DDLDe
boolean ifExists;
boolean ifPurge;
boolean ignoreProtection;
+ ReplicationSpec replicationSpec;
public DropTableDesc() {
}
@@ -64,17 +65,21 @@ public class DropTableDesc extends DDLDe
* @param tableName
* @param ifPurge
*/
- public DropTableDesc(String tableName, boolean expectView, boolean ifExists, boolean ifPurge) {
+ public DropTableDesc(
+ String tableName, boolean expectView, boolean ifExists,
+ boolean ifPurge, ReplicationSpec replicationSpec) {
this.tableName = tableName;
this.partSpecs = null;
this.expectView = expectView;
this.ifExists = ifExists;
this.ifPurge = ifPurge;
this.ignoreProtection = false;
+ this.replicationSpec = replicationSpec;
}
public DropTableDesc(String tableName, Map<Integer, List<ExprNodeGenericFuncDesc>> partSpecs,
- boolean expectView, boolean ignoreProtection, boolean ifPurge) {
+ boolean expectView, boolean ignoreProtection, boolean ifPurge,
+ ReplicationSpec replicationSpec) {
this.tableName = tableName;
this.partSpecs = new ArrayList<PartSpec>(partSpecs.size());
for (Map.Entry<Integer, List<ExprNodeGenericFuncDesc>> partSpec : partSpecs.entrySet()) {
@@ -86,6 +91,7 @@ public class DropTableDesc extends DDLDe
this.ignoreProtection = ignoreProtection;
this.expectView = expectView;
this.ifPurge = ifPurge;
+ this.replicationSpec = replicationSpec;
}
/**
@@ -170,4 +176,15 @@ public class DropTableDesc extends DDLDe
public void setIfPurge(boolean ifPurge) {
this.ifPurge = ifPurge;
}
+
+ /**
+ * @return what kind of replication scope this drop is running under.
+ * This can result in a "DROP IF OLDER THAN" kind of semantic
+ */
+ public ReplicationSpec getReplicationSpec(){
+ if (replicationSpec == null){
+ this.replicationSpec = new ReplicationSpec();
+ }
+ return this.replicationSpec;
+ }
}
Added: hive/trunk/ql/src/test/queries/clientpositive/repl_1_drop.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/repl_1_drop.q?rev=1674557&view=auto==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/repl_1_drop.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/repl_1_drop.q Sat Apr 18 19:04:46 2015
@@ -0,0 +1,84 @@
+set hive.test.mode=true;
+set hive.test.mode.prefix=;
+
+create table repl_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code", emp_state string comment "free text")
+ stored as textfile
+ tblproperties("repl.last.id"="43");
+
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ca");
+
+show partitions repl_employee;
+show table extended like repl_employee;
+
+drop table repl_employee for replication('33');
+
+-- drop 33 => table does not get dropped, but ca will be
+
+show partitions repl_employee;
+show table extended like repl_employee;
+
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ak");
+
+show partitions repl_employee;
+
+drop table repl_employee for replication('');
+
+-- drop '' => ptns would be dropped, but not tables
+
+show partitions repl_employee;
+show table extended like repl_employee;
+
+drop table repl_employee for replication('49');
+
+-- table and ptns should have been dropped, so next create can succeed
+
+create table repl_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code", emp_state string comment "free text")
+ stored as textfile;
+
+-- created table without a repl.last.id
+
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ca");
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ak");
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="wa");
+
+show partitions repl_employee;
+show table extended like repl_employee;
+
+alter table repl_employee drop partition (emp_country="us", emp_state="ca");
+alter table repl_employee drop partition (emp_country="us", emp_state="wa") for replication('59');
+
+-- should have dropped ca, wa
+
+show partitions repl_employee;
+show table extended like repl_employee;
+
+alter table repl_employee set tblproperties ("repl.last.id" = "42");
+
+alter table repl_employee drop partition (emp_country="us", emp_state="ak");
+
+-- should have dropped ak
+
+show partitions repl_employee;
+show table extended like repl_employee;
+
+drop table repl_employee;
+
+-- should drop the whole table, and this can be verified by trying to create another table with the same name
+
+create table repl_employee( a string);
+
+show table extended like repl_employee;
+
+drop table repl_employee;
+
+
+
Added: hive/trunk/ql/src/test/queries/clientpositive/repl_2_exim_basic.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/repl_2_exim_basic.q?rev=1674557&view=auto==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/repl_2_exim_basic.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/repl_2_exim_basic.q Sat Apr 18 19:04:46 2015
@@ -0,0 +1,79 @@
+set hive.test.mode=true;
+set hive.test.mode.prefix=;
+set hive.test.mode.nosamplelist=managed_t,ext_t,managed_t_imported,managed_t_r_imported,ext_t_imported,ext_t_r_imported;
+
+drop table if exists managed_t;
+drop table if exists ext_t;
+drop table if exists managed_t_imported;
+drop table if exists managed_t_r_imported;
+drop table if exists ext_t_imported;
+drop table if exists ext_t_r_imported;
+
+create table managed_t (emp_id int comment "employee id")
+ partitioned by (emp_country string, emp_state string)
+ stored as textfile;
+load data local inpath "../../data/files/test.dat"
+ into table managed_t partition (emp_country="us",emp_state="ca");
+
+create external table ext_t (emp_id int comment "employee id")
+ partitioned by (emp_country string, emp_state string)
+ stored as textfile
+ tblproperties("EXTERNAL"="true");
+load data local inpath "../../data/files/test.dat"
+ into table ext_t partition (emp_country="us",emp_state="ca");
+
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/managed_t/temp;
+dfs -rmr target/tmp/ql/test/data/exports/managed_t;
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/managed_t_r/temp;
+dfs -rmr target/tmp/ql/test/data/exports/managed_t_r;
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/ext_t/temp;
+dfs -rmr target/tmp/ql/test/data/exports/ext_t;
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/ext_t_r/temp;
+dfs -rmr target/tmp/ql/test/data/exports/ext_t_r;
+
+-- verifying difference between normal export of a external table
+-- and a replication export of an ext table
+-- the replication export will have squashed the "EXTERNAL" flag
+-- this is because the destination of all replication exports are
+-- managed tables. The managed tables should be similar except
+-- for the repl.last.id values
+
+export table managed_t to 'ql/test/data/exports/managed_t';
+export table managed_t to 'ql/test/data/exports/managed_t_r' for replication('managed_t_r');
+export table ext_t to 'ql/test/data/exports/ext_t';
+export table ext_t to 'ql/test/data/exports/ext_t_r' for replication('ext_t_r');
+
+drop table ext_t;
+drop table managed_t;
+
+import table managed_t_imported from 'ql/test/data/exports/managed_t';
+describe extended managed_t_imported;
+show table extended like managed_t_imported;
+show create table managed_t_imported;
+select * from managed_t_imported;
+
+-- should have repl.last.id
+import table managed_t_r_imported from 'ql/test/data/exports/managed_t_r';
+describe extended managed_t_r_imported;
+show table extended like managed_t_r_imported;
+show create table managed_t_r_imported;
+select * from managed_t_r_imported;
+
+import table ext_t_imported from 'ql/test/data/exports/ext_t';
+describe extended ext_t_imported;
+show table extended like ext_t_imported;
+show create table ext_t_imported;
+select * from ext_t_imported;
+
+-- should have repl.last.id
+-- also - importing an external table replication export would turn the new table into a managed table
+import table ext_t_r_imported from 'ql/test/data/exports/ext_t_r';
+describe extended ext_t_imported;
+show table extended like ext_t_r_imported;
+show create table ext_t_r_imported;
+select * from ext_t_r_imported;
+
+drop table managed_t_imported;
+drop table managed_t_r_imported;
+drop table ext_t_imported;
+drop table ext_t_r_imported;
Added: hive/trunk/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q?rev=1674557&view=auto==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q Sat Apr 18 19:04:46 2015
@@ -0,0 +1,40 @@
+set hive.test.mode=true;
+set hive.test.mode.prefix=;
+set hive.test.mode.nosamplelist=replsrc,repldst,repldst_md;
+
+drop table if exists replsrc;
+drop table if exists repldst;
+drop table if exists repldst_md;
+
+create table replsrc (emp_id int comment "employee id")
+ partitioned by (emp_country string, emp_state string)
+ stored as textfile;
+load data local inpath "../../data/files/test.dat"
+ into table replsrc partition (emp_country="us",emp_state="ca");
+
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/repldst/temp;
+dfs -rmr target/tmp/ql/test/data/exports/repldst;
+dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/repldst_md/temp;
+dfs -rmr target/tmp/ql/test/data/exports/repldst_md;
+
+export table replsrc to 'ql/test/data/exports/repldst' for replication('repldst');
+export table replsrc to 'ql/test/data/exports/repldst_md' for metadata replication('repldst md-only');
+
+drop table replsrc;
+
+import table repldst from 'ql/test/data/exports/repldst';
+describe extended repldst;
+show table extended like repldst;
+show create table repldst;
+select * from repldst;
+
+-- should be similar, except that select will return no results
+import table repldst_md from 'ql/test/data/exports/repldst_md';
+describe extended repldst_md;
+show table extended like repldst_md;
+show create table repldst_md;
+select * from repldst_md;
+
+drop table repldst;
+drop table repldst_md;
+
Modified: hive/trunk/ql/src/test/results/clientnegative/alter_table_add_partition.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/alter_table_add_partition.q.out?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/alter_table_add_partition.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/alter_table_add_partition.q.out Sat Apr 18 19:04:46 2015
@@ -6,4 +6,4 @@ POSTHOOK: query: create table mp (a int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@mp
-FAILED: SemanticException Partition spec {b=1, c=1} contains non-partition columns
+FAILED: ValidationFailureSemanticException Partition spec {b=1, c=1} contains non-partition columns
Modified: hive/trunk/ql/src/test/results/clientnegative/alter_view_failure5.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/alter_view_failure5.q.out?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/alter_view_failure5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/alter_view_failure5.q.out Sat Apr 18 19:04:46 2015
@@ -18,4 +18,4 @@ POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@xxx6
-FAILED: SemanticException Partition spec {v=val_86} contains non-partition columns
+FAILED: ValidationFailureSemanticException Partition spec {v=val_86} contains non-partition columns
Modified: hive/trunk/ql/src/test/results/clientnegative/alter_view_failure7.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/alter_view_failure7.q.out?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/alter_view_failure7.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/alter_view_failure7.q.out Sat Apr 18 19:04:46 2015
@@ -18,4 +18,4 @@ POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: default@srcpart
POSTHOOK: Output: database:default
POSTHOOK: Output: default@xxx8
-FAILED: SemanticException partition spec {ds=2011-01-01} doesn't contain all (2) partition columns
+FAILED: ValidationFailureSemanticException partition spec {ds=2011-01-01} doesn't contain all (2) partition columns
Modified: hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out?rev=1674557&r1=1674556&r2=1674557&view=diff==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out Sat Apr 18 19:04:46 2015
@@ -20,4 +20,4 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@test_tab@part=1
POSTHOOK: Lineage: test_tab PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: table is partitioned but partition spec is not specified
+FAILED: SemanticException org.apache.hadoop.hive.ql.metadata.Table$ValidationFailureSemanticException: table is partitioned but partition spec is not specified
Added: hive/trunk/ql/src/test/results/clientpositive/repl_1_drop.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/repl_1_drop.q.out?rev=1674557&view=auto==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/repl_1_drop.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/repl_1_drop.q.out Sat Apr 18 19:04:46 2015
@@ -0,0 +1,345 @@
+PREHOOK: query: create table repl_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code", emp_state string comment "free text")
+ stored as textfile
+ tblproperties("repl.last.id"="43")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: create table repl_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code", emp_state string comment "free text")
+ stored as textfile
+ tblproperties("repl.last.id"="43")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ca")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ca")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=ca
+PREHOOK: query: show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+emp_country=us/emp_state=ca
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string emp_state}
+totalNumberFiles:1
+totalFileSize:11
+maxFileSize:11
+minFileSize:11
+#### A masked pattern was here ####
+
+PREHOOK: query: drop table repl_employee for replication('33')
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: drop table repl_employee for replication('33')
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: -- drop 33 => table does not get dropped, but ca will be
+
+show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: -- drop 33 => table does not get dropped, but ca will be
+
+show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string emp_state}
+
+PREHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ak")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ak")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=ak
+PREHOOK: query: show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+emp_country=us/emp_state=ak
+PREHOOK: query: drop table repl_employee for replication('')
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: drop table repl_employee for replication('')
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: -- drop '' => ptns would be dropped, but not tables
+
+show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: -- drop '' => ptns would be dropped, but not tables
+
+show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string emp_state}
+
+PREHOOK: query: drop table repl_employee for replication('49')
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: drop table repl_employee for replication('49')
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: -- table and ptns should have been dropped, so next create can succeed
+
+create table repl_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code", emp_state string comment "free text")
+ stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: -- table and ptns should have been dropped, so next create can succeed
+
+create table repl_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code", emp_state string comment "free text")
+ stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: -- created table without a repl.last.id
+
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ca")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: -- created table without a repl.last.id
+
+load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ca")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=ca
+PREHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ak")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="ak")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=ak
+PREHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="wa")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: load data local inpath "../../data/files/test.dat"
+ into table repl_employee partition (emp_country="us", emp_state="wa")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=wa
+PREHOOK: query: show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+emp_country=us/emp_state=ak
+emp_country=us/emp_state=ca
+emp_country=us/emp_state=wa
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string emp_state}
+totalNumberFiles:3
+totalFileSize:33
+maxFileSize:11
+minFileSize:11
+#### A masked pattern was here ####
+
+PREHOOK: query: alter table repl_employee drop partition (emp_country="us", emp_state="ca")
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee@emp_country=us/emp_state=ca
+POSTHOOK: query: alter table repl_employee drop partition (emp_country="us", emp_state="ca")
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=ca
+PREHOOK: query: alter table repl_employee drop partition (emp_country="us", emp_state="wa") for replication('59')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee@emp_country=us/emp_state=wa
+POSTHOOK: query: alter table repl_employee drop partition (emp_country="us", emp_state="wa") for replication('59')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=wa
+PREHOOK: query: -- should have dropped ca, wa
+
+show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: -- should have dropped ca, wa
+
+show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+emp_country=us/emp_state=ak
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string emp_state}
+totalNumberFiles:1
+totalFileSize:11
+maxFileSize:11
+minFileSize:11
+#### A masked pattern was here ####
+
+PREHOOK: query: alter table repl_employee set tblproperties ("repl.last.id" = "42")
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: alter table repl_employee set tblproperties ("repl.last.id" = "42")
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: alter table repl_employee drop partition (emp_country="us", emp_state="ak")
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee@emp_country=us/emp_state=ak
+POSTHOOK: query: alter table repl_employee drop partition (emp_country="us", emp_state="ak")
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee@emp_country=us/emp_state=ak
+PREHOOK: query: -- should have dropped ak
+
+show partitions repl_employee
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repl_employee
+POSTHOOK: query: -- should have dropped ak
+
+show partitions repl_employee
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repl_employee
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string emp_state}
+
+PREHOOK: query: drop table repl_employee
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: drop table repl_employee
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: -- should drop the whole table, and this can be verified by trying to create another table with the same name
+
+create table repl_employee( a string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: -- should drop the whole table, and this can be verified by trying to create another table with the same name
+
+create table repl_employee( a string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repl_employee
+PREHOOK: query: show table extended like repl_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like repl_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:repl_employee
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { string a}
+partitioned:false
+partitionColumns:
+totalNumberFiles:0
+totalFileSize:0
+maxFileSize:0
+minFileSize:0
+#### A masked pattern was here ####
+
+PREHOOK: query: drop table repl_employee
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repl_employee
+PREHOOK: Output: default@repl_employee
+POSTHOOK: query: drop table repl_employee
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repl_employee
+POSTHOOK: Output: default@repl_employee
