FAQ
Author: jdere
Date: Thu Mar 26 17:55:27 2015
New Revision: 1669378

URL: http://svn.apache.org/r1669378
Log:
HIVE-10072: Add vectorization support for Hybrid Grace Hash Join (Wei Zheng, reviewed by Matt McCline)

Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
     hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q
     hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Mar 26 17:55:27 2015
@@ -404,7 +404,7 @@ public class MapJoinOperator extends Abs
     * @param hybridHtContainer Hybrid hashtable container
     * @param row big table row
     */
- private void spillBigTableRow(MapJoinTableContainer hybridHtContainer, Object row) {
+ protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer, Object row) throws HiveException {
      HybridHashTableContainer ht = (HybridHashTableContainer) hybridHtContainer;
      int partitionId = ht.getToSpillPartitionId();
      HashPartition hp = ht.getHashPartitions()[partitionId];
@@ -469,8 +469,7 @@ public class MapJoinOperator extends Abs
    }

    /**
- * Continue processing each pair of spilled hashtable and big table row container,
- * by bringing them back to memory and calling process() again.
+ * Continue processing each pair of spilled hashtable and big table row container
     * @param partition hash partition to process
     * @param hybridHtContainer Hybrid hashtable container
     * @throws HiveException
@@ -481,13 +480,7 @@ public class MapJoinOperator extends Abs
    private void continueProcess(HashPartition partition, HybridHashTableContainer hybridHtContainer)
        throws HiveException, IOException, ClassNotFoundException, SerDeException {
      reloadHashTable(partition, hybridHtContainer);
- // Iterate thru the on-disk matchfile, and feed processOp with leftover rows
- ObjectContainer bigTable = partition.getMatchfileObjContainer();
- while (bigTable.hasNext()) {
- Object row = bigTable.next();
- process(row, tag);
- }
- bigTable.clear();
+ reProcessBigTable(partition);
    }

    /**
@@ -547,6 +540,20 @@ public class MapJoinOperator extends Abs
    }

    /**
+ * Iterate over the big table row container and feed process() with leftover rows
+ * @param partition the hash partition being brought back to memory at the moment
+ * @throws HiveException
+ */
+ protected void reProcessBigTable(HashPartition partition) throws HiveException {
+ ObjectContainer bigTable = partition.getMatchfileObjContainer();
+ while (bigTable.hasNext()) {
+ Object row = bigTable.next();
+ process(row, tag);
+ }
+ bigTable.clear();
+ }
+
+ /**
     * Implements the getName function for the Node Interface.
     *
     * @return the name of the operator

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java Thu Mar 26 17:55:27 2015
@@ -49,7 +49,7 @@ public class ObjectContainer<ROW> {
    private boolean readBufferUsed = false; // indicates if read buffer has data
    private int rowsInReadBuffer = 0; // number of rows in the temporary read buffer
    private int readCursor = 0; // cursor during reading
- private int rowsOnDisk = 0; // total number of pairs in output
+ private int rowsOnDisk = 0; // total number of objects in output

    private File parentFile;
    private File tmpFile;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java Thu Mar 26 17:55:27 2015
@@ -18,6 +18,7 @@

  package org.apache.hadoop.hive.ql.exec.vector;

+import java.util.Arrays;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.HashMap;
@@ -31,8 +32,10 @@ import org.apache.hadoop.conf.Configurat
  import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
  import org.apache.hadoop.hive.ql.exec.JoinUtil;
  import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer;
  import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
  import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
+import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
  import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
@@ -42,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.Ma
  import org.apache.hadoop.hive.ql.plan.OperatorDesc;
  import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
  import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.DataOutputBuffer;

  /**
   * The vectorized version of the MapJoinOperator.
@@ -68,6 +72,7 @@ public class VectorMapJoinOperator exten
    //---------------------------------------------------------------------------

    private transient VectorizedRowBatch outputBatch;
+ private transient VectorizedRowBatch scratchBatch; // holds restored (from disk) big table rows
    private transient VectorExpressionWriter[] valueWriters;
    private transient Map<ObjectInspector, VectorColumnAssign[]> outputVectorAssigners;

@@ -81,6 +86,10 @@ public class VectorMapJoinOperator exten

    private transient VectorizedRowBatchCtx vrbCtx = null;

+ private transient int tag; // big table alias
+ private VectorExpressionWriter[] rowWriters; // Writer for producing row from input batch
+ protected transient Object[] singleRow;
+
    public VectorMapJoinOperator() {
      super();
    }
@@ -117,6 +126,19 @@ public class VectorMapJoinOperator exten

    @Override
    public Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+ // Code borrowed from VectorReduceSinkOperator.initializeOp
+ VectorExpressionWriterFactory.processVectorInspector(
+ (StructObjectInspector) inputObjInspectors[0],
+ new VectorExpressionWriterFactory.SingleOIDClosure() {
+ @Override
+ public void assign(VectorExpressionWriter[] writers,
+ ObjectInspector objectInspector) {
+ rowWriters = writers;
+ inputObjInspectors[0] = objectInspector;
+ }
+ });
+ singleRow = new Object[rowWriters.length];
+
      Collection<Future<?>> result = super.initializeOp(hconf);

      List<ExprNodeDesc> keyDesc = conf.getKeys().get(posBigTable);
@@ -213,6 +235,7 @@ public class VectorMapJoinOperator exten

    @Override
    public void closeOp(boolean aborted) throws HiveException {
+ super.closeOp(aborted);
      for (MapJoinTableContainer tableContainer : mapJoinTables) {
        if (tableContainer != null) {
          tableContainer.dumpMetrics();
@@ -221,7 +244,6 @@ public class VectorMapJoinOperator exten
      if (!aborted && 0 < outputBatch.size) {
        flushOutput();
      }
- super.closeOp(aborted);
    }

    @Override
@@ -235,6 +257,12 @@ public class VectorMapJoinOperator exten
      byte alias = (byte) tag;
      VectorizedRowBatch inBatch = (VectorizedRowBatch) row;

+ // Preparation for hybrid grace hash join
+ this.tag = tag;
+ if (scratchBatch == null) {
+ scratchBatch = makeLike(inBatch);
+ }
+
      if (null != bigTableFilterExpressions) {
        for(VectorExpression ve:bigTableFilterExpressions) {
          ve.evaluate(inBatch);
@@ -270,4 +298,94 @@ public class VectorMapJoinOperator exten
    public VectorizationContext getOuputVectorizationContext() {
      return vOutContext;
    }
+
+ @Override
+ protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer, Object row)
+ throws HiveException {
+ // Extract the actual row from row batch
+ VectorizedRowBatch inBatch = (VectorizedRowBatch) row;
+ Object[] actualRow = getRowObject(inBatch, batchIndex);
+ super.spillBigTableRow(hybridHtContainer, actualRow);
+ }
+
+ @Override
+ protected void reProcessBigTable(HybridHashTableContainer.HashPartition partition)
+ throws HiveException {
+ ObjectContainer bigTable = partition.getMatchfileObjContainer();
+
+ DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
+ while (bigTable.hasNext()) {
+ Object row = bigTable.next();
+ VectorizedBatchUtil.addProjectedRowToBatchFrom(row,
+ (StructObjectInspector) inputObjInspectors[posBigTable],
+ scratchBatch.size, scratchBatch, dataOutputBuffer);
+ scratchBatch.size++;
+
+ if (scratchBatch.size == VectorizedRowBatch.DEFAULT_SIZE) {
+ process(scratchBatch, tag); // call process once we have a full batch
+ scratchBatch.reset();
+ dataOutputBuffer.reset();
+ }
+ }
+ // Process the row batch that has less than DEFAULT_SIZE rows
+ if (scratchBatch.size > 0) {
+ process(scratchBatch, tag);
+ scratchBatch.reset();
+ dataOutputBuffer.reset();
+ }
+ bigTable.clear();
+ }
+
+ // Code borrowed from VectorReduceSinkOperator
+ private Object[] getRowObject(VectorizedRowBatch vrb, int rowIndex) throws HiveException {
+ int batchIndex = rowIndex;
+ if (vrb.selectedInUse) {
+ batchIndex = vrb.selected[rowIndex];
+ }
+ for (int i = 0; i < vrb.projectionSize; i++) {
+ ColumnVector vectorColumn = vrb.cols[vrb.projectedColumns[i]];
+ if (vectorColumn != null) {
+ singleRow[i] = rowWriters[i].writeValue(vectorColumn, batchIndex);
+ } else {
+ // Some columns from tables are not used.
+ singleRow[i] = null;
+ }
+ }
+ return singleRow;
+ }
+
+ /**
+ * Make a new (scratch) batch, which is exactly "like" the batch provided, except that it's empty
+ * @param batch the batch to imitate
+ * @return the new batch
+ * @throws HiveException
+ */
+ VectorizedRowBatch makeLike(VectorizedRowBatch batch) throws HiveException {
+ VectorizedRowBatch newBatch = new VectorizedRowBatch(batch.numCols);
+ for (int i = 0; i < batch.numCols; i++) {
+ ColumnVector colVector = batch.cols[i];
+ if (colVector != null) {
+ ColumnVector newColVector;
+ if (colVector instanceof LongColumnVector) {
+ newColVector = new LongColumnVector();
+ } else if (colVector instanceof DoubleColumnVector) {
+ newColVector = new DoubleColumnVector();
+ } else if (colVector instanceof BytesColumnVector) {
+ newColVector = new BytesColumnVector();
+ } else if (colVector instanceof DecimalColumnVector) {
+ DecimalColumnVector decColVector = (DecimalColumnVector) colVector;
+ newColVector = new DecimalColumnVector(decColVector.precision, decColVector.scale);
+ } else {
+ throw new HiveException("Column vector class " + colVector.getClass().getName() +
+ " is not supported!");
+ }
+ newBatch.cols[i] = newColVector;
+ newBatch.cols[i].init();
+ }
+ }
+ newBatch.projectedColumns = Arrays.copyOf(batch.projectedColumns, batch.projectedColumns.length);
+ newBatch.projectionSize = batch.projectionSize;
+ newBatch.reset();
+ return newBatch;
+ }
  }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java Thu Mar 26 17:55:27 2015
@@ -242,11 +242,31 @@ public class VectorizedBatchUtil {
      final int off = colOffset;
      // Iterate thru the cols and load the batch
      for (int i = 0; i < fieldRefs.size(); i++) {
- setVector(row, oi, fieldRefs, batch, buffer, rowIndex, i, off);
+ setVector(row, oi, fieldRefs.get(i), batch, buffer, rowIndex, i, off);
      }
    }

    /**
+ * Add only the projected column of a regular row to the specified vectorized row batch
+ * @param row the regular row
+ * @param oi object inspector for the row
+ * @param rowIndex the offset to add in the batch
+ * @param batch vectorized row batch
+ * @param buffer data output buffer
+ * @throws HiveException
+ */
+ public static void addProjectedRowToBatchFrom(Object row, StructObjectInspector oi,
+ int rowIndex, VectorizedRowBatch batch, DataOutputBuffer buffer) throws HiveException {
+ List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
+ for (int i = 0; i < fieldRefs.size(); i++) {
+ int projectedOutputCol = batch.projectedColumns[i];
+ if (batch.cols[projectedOutputCol] == null) {
+ continue;
+ }
+ setVector(row, oi, fieldRefs.get(i), batch, buffer, rowIndex, projectedOutputCol, 0);
+ }
+ }
+ /**
     * Iterates thru all the columns in a given row and populates the batch
     * from a given offset
     *
@@ -275,21 +295,21 @@ public class VectorizedBatchUtil {
          // The value will have already been set before we're called, so don't overwrite it
          continue;
        }
- setVector(row, oi, fieldRefs, batch, buffer, rowIndex, i, 0);
+ setVector(row, oi, fieldRefs.get(i), batch, buffer, rowIndex, i, 0);
      }
    }

    private static void setVector(Object row,
                                  StructObjectInspector oi,
- List<? extends StructField> fieldRefs,
+ StructField field,
                                  VectorizedRowBatch batch,
                                  DataOutputBuffer buffer,
                                  int rowIndex,
                                  int colIndex,
                                  int offset) throws HiveException {

- Object fieldData = oi.getStructFieldData(row, fieldRefs.get(colIndex));
- ObjectInspector foi = fieldRefs.get(colIndex).getFieldObjectInspector();
+ Object fieldData = oi.getStructFieldData(row, field);
+ ObjectInspector foi = field.getFieldObjectInspector();

      // Vectorization only supports PRIMITIVE data types. Assert the same
      assert (foi.getCategory() == Category.PRIMITIVE);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Thu Mar 26 17:55:27 2015
@@ -1301,9 +1301,6 @@ public class Vectorizer implements Physi

      switch (op.getType()) {
        case MAPJOIN:
- // Disable Hybrid Grace Hash Join when vectorization is in effect, for now
- HiveConf.setBoolVar(physicalContext.getConf(),
- HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN, false);
        case GROUPBY:
        case FILTER:
        case SELECT:

Modified: hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q Thu Mar 26 17:55:27 2015
@@ -210,3 +210,41 @@ select count(*) from
  ;

  drop table parttbl;
+
+
+-- Test vectorization
+-- Test case borrowed from vector_decimal_mapjoin.q
+CREATE TABLE decimal_mapjoin STORED AS ORC AS
+ SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1,
+ CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2,
+ cint
+ FROM alltypesorc;
+
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.noconditionaltask=true;
+SET hive.auto.convert.join.noconditionaltask.size=50000000;
+set hive.mapjoin.optimized.hashtable.wbsize=10000;
+SET hive.vectorized.execution.enabled=true;
+set hive.mapjoin.hybridgrace.hashtable=false;
+
+EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+
+set hive.mapjoin.hybridgrace.hashtable=true;
+
+EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+
+DROP TABLE decimal_mapjoin;

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out Thu Mar 26 17:55:27 2015
@@ -1154,3 +1154,407 @@ POSTHOOK: query: drop table parttbl
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@parttbl
  POSTHOOK: Output: default@parttbl
+PREHOOK: query: -- Test vectorization
+-- Test case borrowed from vector_decimal_mapjoin.q
+CREATE TABLE decimal_mapjoin STORED AS ORC AS
+ SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1,
+ CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2,
+ cint
+ FROM alltypesorc
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_mapjoin
+POSTHOOK: query: -- Test vectorization
+-- Test case borrowed from vector_decimal_mapjoin.q
+CREATE TABLE decimal_mapjoin STORED AS ORC AS
+ SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1,
+ CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2,
+ cint
+ FROM alltypesorc
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_mapjoin
+PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: l
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 6981 (type: int)
+ 1 6981 (type: int)
+ outputColumnNames: _col1, _col9
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 6981 (type: int), 6981 (type: int), _col1 (type: decimal(20,10)), _col9 (type: decimal(23,14))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: r
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 6981 (type: int)
+ sort order: +
+ Map-reduce partition columns: 6981 (type: int)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
+ value expressions: cdecimal2 (type: decimal(23,14))
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 6984454.211097692
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: l
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 6981 (type: int)
+ 1 6981 (type: int)
+ outputColumnNames: _col1, _col9
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 6981 (type: int), 6981 (type: int), _col1 (type: decimal(20,10)), _col9 (type: decimal(23,14))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: r
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 6981 (type: int)
+ sort order: +
+ Map-reduce partition columns: 6981 (type: int)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
+ value expressions: cdecimal2 (type: decimal(23,14))
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 6984454.211097692
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+PREHOOK: query: DROP TABLE decimal_mapjoin
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_mapjoin
+PREHOOK: Output: default@decimal_mapjoin
+POSTHOOK: query: DROP TABLE decimal_mapjoin
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_mapjoin
+POSTHOOK: Output: default@decimal_mapjoin

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 26, '15 at 5:55p
activeMar 26, '15 at 5:55p
posts1
users1
websitehive.apache.org

1 user in discussion

Jdere: 1 post

People

Translate

site design / logo © 2021 Grokbase