Grokbase Groups Hive commits May 2012
FAQ
Author: kevinwilfong
Date: Thu May 31 16:31:53 2012
New Revision: 1344801

URL: http://svn.apache.org/viewvc?rev=1344801&view=rev
Log:
HIVE-3063. drop partition for non-string columns is failing (njain via kevinwilfong)

Added:
hive/trunk/ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q
hive/trunk/ql/src/test/queries/clientpositive/drop_partitions_filter2.q
hive/trunk/ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out
hive/trunk/ql/src/test/results/clientpositive/drop_partitions_filter2.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1344801&r1=1344800&r2=1344801&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Thu May 31 16:31:53 2012
@@ -213,6 +213,8 @@ public enum ErrorMsg {
INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS(10127,
"Dynamic partitions do not support IF NOT EXISTS. Specified partitions with value :"),
UDAF_INVALID_LOCATION(10128, "Not yet supported place for UDAF"),
+ DROP_PARTITION_NON_STRING_PARTCOLS_NONEQUALITY(10129,
+ "Drop partitions for a non string partition columns is not allowed using non-equality"),

SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1344801&r1=1344800&r2=1344801&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Thu May 31 16:31:53 2012
@@ -3086,11 +3086,19 @@ public class DDLTask extends Task<DDLWor
List<Partition> partsToDelete = new ArrayList<Partition>();
for (PartitionSpec partSpec : dropTbl.getPartSpecs()) {
List<Partition> partitions = null;
- try {
- partitions = db.getPartitionsByFilter(tbl, partSpec.toString());
- } catch (Exception e) {
- throw new HiveException(e);
+ // getPartitionsByFilter only works for string columns.
+ // Till that is fixed, only equality will work for non-string columns.
+ if (dropTbl.isStringPartitionColumns()) {
+ try {
+ partitions = db.getPartitionsByFilter(tbl, partSpec.toString());
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ }
+ else {
+ partitions = db.getPartitions(tbl, partSpec.getPartSpecWithoutOperator());
}
+
// this is to prevent dropping archived partition which is archived in a
// different level the drop command specified.
int partPrefixToDrop = 0;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1344801&r1=1344800&r2=1344801&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Thu May 31 16:31:53 2012
@@ -713,7 +713,7 @@ public class DDLSemanticAnalyzer extends
}

DropTableDesc dropTblDesc = new DropTableDesc(
- tableName, expectView, ifExists);
+ tableName, expectView, ifExists, true);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
dropTblDesc), conf));
}
@@ -1790,11 +1790,10 @@ public class DDLSemanticAnalyzer extends
String tblName = getUnescapedName((ASTNode)ast.getChild(0));
// get table metadata
List<PartitionSpec> partSpecs = getFullPartitionSpecs(ast);
- DropTableDesc dropTblDesc =
- new DropTableDesc(tblName, partSpecs, expectView);
+ Table tab = null;

try {
- Table tab = db.getTable(db.getCurrentDatabase(), tblName, false);
+ tab = db.getTable(db.getCurrentDatabase(), tblName, false);
if (tab != null) {
inputs.add(new ReadEntity(tab));
}
@@ -1802,15 +1801,39 @@ public class DDLSemanticAnalyzer extends
throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName));
}

+ // Find out if all partition columns are strings. This is needed for JDO
+ boolean stringPartitionColumns = true;
+ List<FieldSchema> partCols = tab.getPartCols();
+
+ for (FieldSchema partCol : partCols) {
+ if (!partCol.getType().toLowerCase().equals("string")) {
+ stringPartitionColumns = false;
+ break;
+ }
+ }
+
+ // Only equality is supported for non-string partition columns
+ if (!stringPartitionColumns) {
+ for (PartitionSpec partSpec : partSpecs) {
+ if (partSpec.isNonEqualityOperator()) {
+ throw new SemanticException(
+ ErrorMsg.DROP_PARTITION_NON_STRING_PARTCOLS_NONEQUALITY.getMsg());
+ }
+ }
+ }
+
if (partSpecs != null) {
boolean ifExists = (ast.getFirstChildWithType(TOK_IFEXISTS) != null);
// we want to signal an error if the partition doesn't exist and we're
// configured not to fail silently
boolean throwException =
!ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT);
- addTableDropPartsOutputs(tblName, partSpecs, throwException);
+ addTableDropPartsOutputs(tblName, partSpecs, throwException, stringPartitionColumns);
}

+ DropTableDesc dropTblDesc =
+ new DropTableDesc(tblName, partSpecs, expectView, stringPartitionColumns);
+
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
dropTblDesc), conf));
}
@@ -2218,7 +2241,7 @@ public class DDLSemanticAnalyzer extends
* throwIfNonExistent is true, otherwise ignore it.
*/
private void addTableDropPartsOutputs(String tblName, List<PartitionSpec> partSpecs,
- boolean throwIfNonExistent)
+ boolean throwIfNonExistent, boolean stringPartitionColumns)
throws SemanticException {
Table tab;
try {
@@ -2232,11 +2255,21 @@ public class DDLSemanticAnalyzer extends
for (i = partSpecs.iterator(), index = 1; i.hasNext(); ++index) {
PartitionSpec partSpec = i.next();
List<Partition> parts = null;
- try {
- parts = db.getPartitionsByFilter(tab, partSpec.toString());
- } catch (Exception e) {
- throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()), e);
+ if (stringPartitionColumns) {
+ try {
+ parts = db.getPartitionsByFilter(tab, partSpec.toString());
+ } catch (Exception e) {
+ throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()), e);
+ }
}
+ else {
+ try {
+ parts = db.getPartitions(tab, partSpec.getPartSpecWithoutOperator());
+ } catch (Exception e) {
+ throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()), e);
+ }
+ }
+
if (parts.isEmpty()) {
if(throwIfNonExistent) {
throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java?rev=1344801&r1=1344800&r2=1344801&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java Thu May 31 16:31:53 2012
@@ -34,6 +34,9 @@ public class DropTableDesc extends DDLDe
ArrayList<PartitionSpec> partSpecs;
boolean expectView;
boolean ifExists;
+ boolean stringPartitionColumns; // This is due to JDO not working very well with
+ // non-string partition columns.
+ // We need a different codepath for them

public DropTableDesc() {
}
@@ -41,14 +44,17 @@ public class DropTableDesc extends DDLDe
/**
* @param tableName
*/
- public DropTableDesc(String tableName, boolean expectView, boolean ifExists) {
+ public DropTableDesc(String tableName, boolean expectView,
+ boolean ifExists, boolean stringPartitionColumns) {
this.tableName = tableName;
partSpecs = null;
this.expectView = expectView;
this.ifExists = ifExists;
+ this.stringPartitionColumns = stringPartitionColumns;
}

- public DropTableDesc(String tableName, List<PartitionSpec> partSpecs, boolean expectView) {
+ public DropTableDesc(String tableName, List<PartitionSpec> partSpecs,
+ boolean expectView, boolean stringPartitionColumns) {

this.tableName = tableName;
this.partSpecs = new ArrayList<PartitionSpec>(partSpecs.size());
@@ -56,6 +62,7 @@ public class DropTableDesc extends DDLDe
this.partSpecs.add(partSpecs.get(i));
}
this.expectView = expectView;
+ this.stringPartitionColumns = stringPartitionColumns;
}

/**
@@ -118,4 +125,12 @@ public class DropTableDesc extends DDLDe
public void setIfExists(boolean ifExists) {
this.ifExists = ifExists;
}
+
+ public boolean isStringPartitionColumns() {
+ return stringPartitionColumns;
+ }
+
+ public void setStringPartitionColumns(boolean stringPartitionColumns) {
+ this.stringPartitionColumns = stringPartitionColumns;
+ }
}

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java?rev=1344801&r1=1344800&r2=1344801&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java Thu May 31 16:31:53 2012
@@ -18,6 +18,8 @@

package org.apache.hadoop.hive.ql.plan;

+import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;

@@ -62,7 +64,7 @@ public class PartitionSpec {
}
}

- private Map<String, PredicateSpec> partSpec;
+ private final Map<String, PredicateSpec> partSpec;

public PartitionSpec() {
this.partSpec = new LinkedHashMap<String, PredicateSpec>();
@@ -102,4 +104,29 @@ public class PartitionSpec {
}
return filterString.toString();
}
+
+ // getParitionsByFilter only works for string columns due to a JDO limitation.
+ // The operator is only useful if it can be passed as a filter to the metastore.
+ // For compatibility with other non-string partition columns, this function
+ // returns the key, value mapping assuming that the operator is equality.
+ public Map<String, String> getPartSpecWithoutOperator() {
+ Map<String, String> partSpec = new HashMap<String, String>();
+ for (Map.Entry<String, PredicateSpec> entry: this.partSpec.entrySet()) {
+ partSpec.put(entry.getKey(), entry.getValue().getValue());
+ }
+
+ return partSpec;
+ }
+
+ // Again, for the same reason as the above function - getPartSpecWithoutOperator
+ public boolean isNonEqualityOperator() {
+ Iterator<PredicateSpec> iter = partSpec.values().iterator();
+ while (iter.hasNext()) {
+ PredicateSpec predSpec = iter.next();
+ if (!predSpec.operator.equals("=")) {
+ return true;
+ }
+ }
+ return false;
+ }
}

Added: hive/trunk/ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q?rev=1344801&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q Thu May 31 16:31:53 2012
@@ -0,0 +1,11 @@
+create table ptestfilter (a string, b int) partitioned by (c string, d int);
+describe ptestfilter;
+
+alter table ptestfilter add partition (c='US', d=1);
+alter table ptestfilter add partition (c='US', d=2);
+show partitions ptestfilter;
+
+alter table ptestfilter drop partition (c='US', d<'2');
+
+
+

Added: hive/trunk/ql/src/test/queries/clientpositive/drop_partitions_filter2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/drop_partitions_filter2.q?rev=1344801&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/drop_partitions_filter2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/drop_partitions_filter2.q Thu May 31 16:31:53 2012
@@ -0,0 +1,20 @@
+create table ptestfilter (a string, b int) partitioned by (c int, d int);
+describe ptestfilter;
+
+alter table ptestfilter add partition (c=1, d=1);
+alter table ptestfilter add partition (c=1, d=2);
+alter table ptestFilter add partition (c=2, d=1);
+alter table ptestfilter add partition (c=2, d=2);
+alter table ptestfilter add partition (c=3, d=1);
+alter table ptestfilter add partition (c=3, d=2);
+show partitions ptestfilter;
+
+alter table ptestfilter drop partition (c=1, d=1);
+show partitions ptestfilter;
+
+alter table ptestfilter drop partition (c=2);
+show partitions ptestfilter;
+
+drop table ptestfilter;
+
+

Added: hive/trunk/ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out?rev=1344801&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out Thu May 31 16:31:53 2012
@@ -0,0 +1,34 @@
+PREHOOK: query: create table ptestfilter (a string, b int) partitioned by (c string, d int)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table ptestfilter (a string, b int) partitioned by (c string, d int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ptestfilter
+PREHOOK: query: describe ptestfilter
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe ptestfilter
+POSTHOOK: type: DESCTABLE
+a string
+b int
+c string
+d int
+PREHOOK: query: alter table ptestfilter add partition (c='US', d=1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c='US', d=1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=US/d=1
+PREHOOK: query: alter table ptestfilter add partition (c='US', d=2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c='US', d=2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=US/d=2
+PREHOOK: query: show partitions ptestfilter
+PREHOOK: type: SHOWPARTITIONS
+POSTHOOK: query: show partitions ptestfilter
+POSTHOOK: type: SHOWPARTITIONS
+c=US/d=1
+c=US/d=2
+FAILED: SemanticException [Error 10129]: Drop partitions for a non string partition columns is not allowed using non-equality

Added: hive/trunk/ql/src/test/results/clientpositive/drop_partitions_filter2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/drop_partitions_filter2.q.out?rev=1344801&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/drop_partitions_filter2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/drop_partitions_filter2.q.out Thu May 31 16:31:53 2012
@@ -0,0 +1,107 @@
+PREHOOK: query: create table ptestfilter (a string, b int) partitioned by (c int, d int)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table ptestfilter (a string, b int) partitioned by (c int, d int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ptestfilter
+PREHOOK: query: describe ptestfilter
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe ptestfilter
+POSTHOOK: type: DESCTABLE
+a string
+b int
+c int
+d int
+PREHOOK: query: alter table ptestfilter add partition (c=1, d=1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c=1, d=1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=1/d=1
+PREHOOK: query: alter table ptestfilter add partition (c=1, d=2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c=1, d=2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=1/d=2
+PREHOOK: query: alter table ptestFilter add partition (c=2, d=1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestFilter add partition (c=2, d=1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=2/d=1
+PREHOOK: query: alter table ptestfilter add partition (c=2, d=2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c=2, d=2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=2/d=2
+PREHOOK: query: alter table ptestfilter add partition (c=3, d=1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c=3, d=1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=3/d=1
+PREHOOK: query: alter table ptestfilter add partition (c=3, d=2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ptestfilter
+POSTHOOK: query: alter table ptestfilter add partition (c=3, d=2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=3/d=2
+PREHOOK: query: show partitions ptestfilter
+PREHOOK: type: SHOWPARTITIONS
+POSTHOOK: query: show partitions ptestfilter
+POSTHOOK: type: SHOWPARTITIONS
+c=1/d=1
+c=1/d=2
+c=2/d=1
+c=2/d=2
+c=3/d=1
+c=3/d=2
+PREHOOK: query: alter table ptestfilter drop partition (c=1, d=1)
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@ptestfilter
+PREHOOK: Output: default@ptestfilter@c=1/d=1
+POSTHOOK: query: alter table ptestfilter drop partition (c=1, d=1)
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=1/d=1
+PREHOOK: query: show partitions ptestfilter
+PREHOOK: type: SHOWPARTITIONS
+POSTHOOK: query: show partitions ptestfilter
+POSTHOOK: type: SHOWPARTITIONS
+c=1/d=2
+c=2/d=1
+c=2/d=2
+c=3/d=1
+c=3/d=2
+PREHOOK: query: alter table ptestfilter drop partition (c=2)
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@ptestfilter
+PREHOOK: Output: default@ptestfilter@c=2/d=1
+PREHOOK: Output: default@ptestfilter@c=2/d=2
+POSTHOOK: query: alter table ptestfilter drop partition (c=2)
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter@c=2/d=1
+POSTHOOK: Output: default@ptestfilter@c=2/d=2
+PREHOOK: query: show partitions ptestfilter
+PREHOOK: type: SHOWPARTITIONS
+POSTHOOK: query: show partitions ptestfilter
+POSTHOOK: type: SHOWPARTITIONS
+c=1/d=2
+c=3/d=1
+c=3/d=2
+PREHOOK: query: drop table ptestfilter
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ptestfilter
+PREHOOK: Output: default@ptestfilter
+POSTHOOK: query: drop table ptestfilter
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ptestfilter
+POSTHOOK: Output: default@ptestfilter

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 31, '12 at 4:32p
activeMay 31, '12 at 4:32p
posts1
users1
websitehive.apache.org

1 user in discussion

Kevinwilfong: 1 post

People

Translate

site design / logo © 2021 Grokbase