FAQ
Author: ehans
Date: Mon Jan 13 19:44:06 2014
New Revision: 1557826

URL: http://svn.apache.org/r1557826
Log:
HIVE-6067: Implement vectorized decimal comparison filters (Eric Hanson)

Added:
     hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt
     hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt
     hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt
Modified:
     hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java

Modified: hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1557826&r1=1557825&r2=1557826&view=diff
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Mon Jan 13 19:44:06 2014
@@ -235,6 +235,27 @@ public class GenVectorCode extends Task
        {"FilterStringScalarCompareColumn", "Greater", ">"},
        {"FilterStringScalarCompareColumn", "GreaterEqual", ">="},

+ {"FilterDecimalColumnCompareScalar", "Equal", "=="},
+ {"FilterDecimalColumnCompareScalar", "NotEqual", "!="},
+ {"FilterDecimalColumnCompareScalar", "Less", "<"},
+ {"FilterDecimalColumnCompareScalar", "LessEqual", "<="},
+ {"FilterDecimalColumnCompareScalar", "Greater", ">"},
+ {"FilterDecimalColumnCompareScalar", "GreaterEqual", ">="},
+
+ {"FilterDecimalScalarCompareColumn", "Equal", "=="},
+ {"FilterDecimalScalarCompareColumn", "NotEqual", "!="},
+ {"FilterDecimalScalarCompareColumn", "Less", "<"},
+ {"FilterDecimalScalarCompareColumn", "LessEqual", "<="},
+ {"FilterDecimalScalarCompareColumn", "Greater", ">"},
+ {"FilterDecimalScalarCompareColumn", "GreaterEqual", ">="},
+
+ {"FilterDecimalColumnCompareColumn", "Equal", "=="},
+ {"FilterDecimalColumnCompareColumn", "NotEqual", "!="},
+ {"FilterDecimalColumnCompareColumn", "Less", "<"},
+ {"FilterDecimalColumnCompareColumn", "LessEqual", "<="},
+ {"FilterDecimalColumnCompareColumn", "Greater", ">"},
+ {"FilterDecimalColumnCompareColumn", "GreaterEqual", ">="},
+
        {"StringScalarCompareColumn", "Equal", "=="},
        {"StringScalarCompareColumn", "NotEqual", "!="},
        {"StringScalarCompareColumn", "Less", "<"},
@@ -593,6 +614,12 @@ public class GenVectorCode extends Task
          generateIfExprScalarColumn(tdesc);
        } else if (tdesc[0].equals("IfExprScalarScalar")) {
          generateIfExprScalarScalar(tdesc);
+ } else if (tdesc[0].equals("FilterDecimalColumnCompareScalar")) {
+ generateFilterDecimalColumnCompareScalar(tdesc);
+ } else if (tdesc[0].equals("FilterDecimalScalarCompareColumn")) {
+ generateFilterDecimalScalarCompareColumn(tdesc);
+ } else if (tdesc[0].equals("FilterDecimalColumnCompareColumn")) {
+ generateFilterDecimalColumnCompareColumn(tdesc);
        } else {
          continue;
        }
@@ -1126,6 +1153,39 @@ public class GenVectorCode extends Task
      generateScalarBinaryOperatorColumn(tdesc, returnType, className);
    }

+ private void generateFilterDecimalColumnCompareScalar(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "FilterDecimalCol" + operatorName + "DecimalScalar";
+ generateDecimalColumnCompare(tdesc, className);
+ }
+
+ private void generateFilterDecimalScalarCompareColumn(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "FilterDecimalScalar" + operatorName + "DecimalColumn";
+ generateDecimalColumnCompare(tdesc, className);
+ }
+
+ private void generateFilterDecimalColumnCompareColumn(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "FilterDecimalCol" + operatorName + "DecimalColumn";
+ generateDecimalColumnCompare(tdesc, className);
+ }
+
+ private void generateDecimalColumnCompare(String[] tdesc, String className)
+ throws IOException {
+ String operatorSymbol = tdesc[2];
+
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
    static void writeFile(long templateTime, String outputDir, String classesDir,
         String className, String str) throws IOException {
      File outputFile = new File(outputDir, className + ".java");

Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt?rev=1557826&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt Mon Jan 13 19:44:06 2014
@@ -0,0 +1,461 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+/**
+ * Generated from template FilterDecimalColumnCompareColumn.txt, which covers binary comparison
+ * filter expressions between two columns. Output is not produced in a separate column.
+ * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum1;
+ private int colNum2;
+
+ public <ClassName>(int colNum1, int colNum2) {
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum1];
+ DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum2];
+ int[] sel = batch.selected;
+ boolean[] nullPos1 = inputColVector1.isNull;
+ boolean[] nullPos2 = inputColVector2.isNull;
+ int n = batch.size;
+ Decimal128[] vector1 = inputColVector1.vector;
+ Decimal128[] vector2 = inputColVector2.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ // handle case where neither input has nulls
+ if (inputColVector1.noNulls && inputColVector2.noNulls) {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+
+ /* Either all must remain selected or all will be eliminated.
+ * Repeating property will not change.
+ */
+ if (!(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ }
+ } else if (inputColVector1.isRepeating) {
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+
+ // handle case where only input 2 has nulls
+ } else if (inputColVector1.noNulls) {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ if (nullPos2[0] ||
+ !(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ }
+ } else if (inputColVector1.isRepeating) {
+
+ // no need to check for nulls in input 1
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos2[i]) {
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos2[i]) {
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (nullPos2[0]) {
+
+ // no values will qualify because every comparison will be with NULL
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else { // neither input is repeating
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos2[i]) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos2[i]) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+
+ // handle case where only input 1 has nulls
+ } else if (inputColVector2.noNulls) {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ if (nullPos1[0] ||
+ !(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ return;
+ }
+ } else if (inputColVector1.isRepeating) {
+ if (nullPos1[0]) {
+
+ // if repeating value is null then every comparison will fail so nothing qualifies
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i]) {
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i]) {
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else { // neither input is repeating
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i]) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i]) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+
+ // handle case where both inputs have nulls
+ } else {
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ if (nullPos1[0] || nullPos2[0] ||
+ !(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+ batch.size = 0;
+ }
+ } else if (inputColVector1.isRepeating) {
+ if (nullPos1[0]) {
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos2[i]) {
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos2[i]) {
+ if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ if (nullPos2[0]) {
+ batch.size = 0;
+ return;
+ }
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i]) {
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i]) {
+ if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else { // neither input is repeating
+ if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos1[i] && !nullPos2[i]) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos1[i] && !nullPos2[i]) {
+ if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < batch.size) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ public int getColNum1() {
+ return colNum1;
+ }
+
+ public void setColNum1(int colNum1) {
+ this.colNum1 = colNum1;
+ }
+
+ public int getColNum2() {
+ return colNum2;
+ }
+
+ public void setColNum2(int colNum2) {
+ this.colNum2 = colNum2;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.FILTER)
+ .setNumArguments(2)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}

Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt?rev=1557826&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt Mon Jan 13 19:44:06 2014
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+/**
+ * This is a generated class to evaluate a <OperatorSymbol> comparison on a vector of decimal
+ * values.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum;
+ private Decimal128 value;
+
+ public <ClassName>(int colNum, Decimal128 value) {
+ this.colNum = colNum;
+ this.value = value;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+ DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum];
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector.isNull;
+ int n = batch.size;
+ Decimal128[] vector = inputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector.noNulls) {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero. Repeating property will not change.
+ if (!(vector[0].compareTo(value) <OperatorSymbol> 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero. Repeating property will not change.
+ if (!nullPos[0]) {
+ if (!(vector[0].compareTo(value) <OperatorSymbol> 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else {
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+
+ // Change the selected vector
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ public int getColNum() {
+ return colNum;
+ }
+
+ public void setColNum(int colNum) {
+ this.colNum = colNum;
+ }
+
+ public Decimal128 getValue() {
+ return value;
+ }
+
+ public void setValue(Decimal128 value) {
+ this.value = value;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.FILTER)
+ .setNumArguments(2)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+ }
+}

Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt?rev=1557826&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt Mon Jan 13 19:44:06 2014
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+/**
+ * This is a generated class to evaluate a <OperatorSymbol> comparison on a vector of decimal
+ * values.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum;
+ private Decimal128 value;
+
+ public <ClassName>(Decimal128 value, int colNum) {
+ this.colNum = colNum;
+ this.value = value;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+ DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum];
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector.isNull;
+ int n = batch.size;
+ Decimal128[] vector = inputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector.noNulls) {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero. Repeating property will not change.
+ if (!(value.compareTo(vector[0]) <OperatorSymbol> 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero. Repeating property will not change.
+ if (!nullPos[0]) {
+ if (!(value.compareTo(vector[0]) <OperatorSymbol> 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else {
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+
+ // Change the selected vector
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ public int getColNum() {
+ return colNum;
+ }
+
+ public void setColNum(int colNum) {
+ this.colNum = colNum;
+ }
+
+ public Decimal128 getValue() {
+ return value;
+ }
+
+ public void setValue(Decimal128 value) {
+ this.value = value;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.FILTER)
+ .setNumArguments(2)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java?rev=1557826&r1=1557825&r2=1557826&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java Mon Jan 13 19:44:06 2014
@@ -24,11 +24,16 @@ import static org.junit.Assert.assertTru

  import java.sql.Timestamp;

+import org.apache.hadoop.hive.common.type.Decimal128;
  import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
  import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
  import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
  import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
  import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColGreaterEqualDecimalColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColLessDecimalScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarGreaterDecimalColumn;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColEqualLongScalar;
@@ -41,6 +46,9 @@ import org.apache.hadoop.hive.ql.exec.ve
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarLessLongColumn;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColEqualDecimalScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColEqualDecimalColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarEqualDecimalColumn;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongScalar;
  import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil;
  import org.junit.Assert;
@@ -784,4 +792,237 @@ public class TestVectorFilterExpressions
      expr.evaluate(vrb);
      assertEquals(0, vrb.size);
    }
+
+ /**
+ * This tests the template for Decimal Column-Scalar comparison filters,
+ * called FilterDecimalColumnCompareScalar.txt. Only equal is tested for
+ * multiple cases because the logic is the same for <, >, <=, >=, == and !=.
+ */
+ @Test
+ public void testFilterDecimalColEqualDecimalScalar() {
+ VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+ Decimal128 scalar = new Decimal128();
+ scalar.update("-3.30", (short) 2);
+ VectorExpression expr = new FilterDecimalColEqualDecimalScalar(0, scalar);
+ expr.evaluate(b);
+
+ // check that right row(s) are selected
+ assertTrue(b.selectedInUse);
+ assertEquals(1, b.selected[0]);
+ assertEquals(1, b.size);
+
+ // try again with a null value
+ b = getVectorizedRowBatch1DecimalCol();
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[1] = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try the repeating case
+ b = getVectorizedRowBatch1DecimalCol();
+ b.cols[0].isRepeating = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try the repeating null case
+ b = getVectorizedRowBatch1DecimalCol();
+ b.cols[0].isRepeating = true;
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[0] = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+ }
+
+ /**
+ * This tests the template for Decimal Scalar-Column comparison filters,
+ * called FilterDecimalScalarCompareColumn.txt. Only equal is tested for multiple
+ * cases because the logic is the same for <, >, <=, >=, == and !=.
+ */
+ @Test
+ public void testFilterDecimalScalarEqualDecimalColumn() {
+ VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+ Decimal128 scalar = new Decimal128();
+ scalar.update("-3.30", (short) 2);
+ VectorExpression expr = new FilterDecimalScalarEqualDecimalColumn(scalar, 0);
+ expr.evaluate(b);
+
+ // check that right row(s) are selected
+ assertTrue(b.selectedInUse);
+ assertEquals(1, b.selected[0]);
+ assertEquals(1, b.size);
+
+ // try again with a null value
+ b = getVectorizedRowBatch1DecimalCol();
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[1] = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try the repeating case
+ b = getVectorizedRowBatch1DecimalCol();
+ b.cols[0].isRepeating = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try the repeating null case
+ b = getVectorizedRowBatch1DecimalCol();
+ b.cols[0].isRepeating = true;
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[0] = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+ }
+
+ /**
+ * This tests the template for Decimal Column-Column comparison filters,
+ * called FilterDecimalColumnCompareColumn.txt. Only equal is tested for multiple
+ * cases because the logic is the same for <, >, <=, >=, == and !=.
+ */
+ @Test
+ public void testFilterDecimalColumnEqualDecimalColumn() {
+ VectorizedRowBatch b = getVectorizedRowBatch2DecimalCol();
+ VectorExpression expr = new FilterDecimalColEqualDecimalColumn(0, 1);
+ expr.evaluate(b);
+
+ // check that right row(s) are selected
+ assertTrue(b.selectedInUse);
+ assertEquals(1, b.selected[0]);
+ assertEquals(1, b.size);
+
+ // try again with a null value
+ b = getVectorizedRowBatch2DecimalCol();
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[1] = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try the repeating case
+ b = getVectorizedRowBatch2DecimalCol();
+ b.cols[0].isRepeating = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try the repeating null case
+ b = getVectorizedRowBatch2DecimalCol();
+ b.cols[0].isRepeating = true;
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[0] = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+
+ // try nulls on both sides
+ b = getVectorizedRowBatch2DecimalCol();
+ b.cols[0].noNulls = false;
+ b.cols[0].isNull[0] = true;
+ b.cols[1].noNulls = false;
+ b.cols[1].isNull[2] = true;
+ expr.evaluate(b);
+ assertEquals(1, b.size); // second of three was selected
+
+ // try repeating on both sides
+ b = getVectorizedRowBatch2DecimalCol();
+ b.cols[0].isRepeating = true;
+ b.cols[1].isRepeating = true;
+ expr.evaluate(b);
+
+ // verify that no rows were selected
+ assertEquals(0, b.size);
+ }
+
+ /**
+ * Spot check col < scalar for decimal.
+ */
+ @Test
+ public void testFilterDecimalColLessScalar() {
+ VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+ Decimal128 scalar = new Decimal128();
+ scalar.update("0", (short) 2);
+ VectorExpression expr = new FilterDecimalColLessDecimalScalar(0, scalar);
+ expr.evaluate(b);
+
+ // check that right row(s) are selected
+ assertTrue(b.selectedInUse);
+ assertEquals(1, b.selected[0]);
+ assertEquals(1, b.size);
+ }
+
+ /**
+ * Spot check scalar > col for decimal.
+ */
+ @Test
+ public void testFilterDecimalScalarGreaterThanColumn() {
+ VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+ Decimal128 scalar = new Decimal128();
+ scalar.update("0", (short) 2);
+ VectorExpression expr = new FilterDecimalScalarGreaterDecimalColumn(scalar, 0);
+ expr.evaluate(b);
+
+ // check that right row(s) are selected
+ assertTrue(b.selectedInUse);
+ assertEquals(1, b.selected[0]);
+ assertEquals(1, b.size);
+ }
+
+ /**
+ * Spot check col >= col for decimal.
+ */
+ @Test
+ public void testFilterDecimalColGreaterEqualCol() {
+ VectorizedRowBatch b = getVectorizedRowBatch2DecimalCol();
+ VectorExpression expr = new FilterDecimalColGreaterEqualDecimalColumn(0, 1);
+ expr.evaluate(b);
+
+ // check that right row(s) are selected
+ assertTrue(b.selectedInUse);
+ assertEquals(0, b.selected[0]);
+ assertEquals(1, b.selected[1]);
+ assertEquals(2, b.size);
+ }
+
+ private VectorizedRowBatch getVectorizedRowBatch1DecimalCol() {
+ VectorizedRowBatch b = new VectorizedRowBatch(1);
+ DecimalColumnVector v0;
+ b.cols[0] = v0 = new DecimalColumnVector(18, 2);
+ v0.vector[0].update("1.20", (short) 2);
+ v0.vector[1].update("-3.30", (short) 2);
+ v0.vector[2].update("0", (short) 2);
+
+ b.size = 3;
+ return b;
+ }
+
+ private VectorizedRowBatch getVectorizedRowBatch2DecimalCol() {
+ VectorizedRowBatch b = new VectorizedRowBatch(2);
+ DecimalColumnVector v0, v1;
+ b.cols[0] = v0 = new DecimalColumnVector(18, 2);
+ v0.vector[0].update("1.20", (short) 2);
+ v0.vector[1].update("-3.30", (short) 2);
+ v0.vector[2].update("0", (short) 2);
+
+ b.cols[1] = v1 = new DecimalColumnVector(18, 2);
+ v1.vector[0].update("-1", (short) 2);
+ v1.vector[1].update("-3.30", (short) 2);
+ v1.vector[2].update("10", (short) 2);
+
+ b.size = 3;
+ return b;
+ }
  }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJan 13, '14 at 8:43p
activeJan 13, '14 at 8:43p
posts1
users1
websitehive.apache.org

1 user in discussion

Ehans: 1 post

People

Translate

site design / logo © 2021 Grokbase