FAQ
Author: xuefu
Date: Thu Sep 25 14:39:02 2014
New Revision: 1627552

URL: http://svn.apache.org/r1627552
Log:
HIVE-8106: Enable vectorization for spark [spark branch] (Chinna via Xuefu)

Added:
     hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
     hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
Modified:
     hive/branches/spark/itests/src/test/resources/testconfiguration.properties
     hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1627552&r1=1627551&r2=1627552&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Thu Sep 25 14:39:02 2014
@@ -547,4 +547,6 @@ spark.query.files=add_part_multiple.q \
    load_dyn_part12.q \
    load_dyn_part13.q \
    load_dyn_part14.q \
- load_dyn_part15.q
+ load_dyn_part15.q \
+ vectorization_13.q \
+ vectorized_shufflejoin.q

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1627552&r1=1627551&r2=1627552&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Thu Sep 25 14:39:02 2014
@@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFac
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.ql.exec.*;
  import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
  import org.apache.hadoop.hive.ql.exec.tez.TezTask;
  import org.apache.hadoop.hive.ql.exec.vector.VectorExtractOperator;
  import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
@@ -71,6 +72,7 @@ import org.apache.hadoop.hive.ql.plan.Op
  import org.apache.hadoop.hive.ql.plan.PartitionDesc;
  import org.apache.hadoop.hive.ql.plan.ReduceWork;
  import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
+import org.apache.hadoop.hive.ql.plan.SparkWork;
  import org.apache.hadoop.hive.ql.plan.TableDesc;
  import org.apache.hadoop.hive.ql.plan.TableScanDesc;
  import org.apache.hadoop.hive.ql.plan.TezWork;
@@ -308,6 +310,15 @@ public class Vectorizer implements Physi
              }
            }
          }
+ } else if (currTask instanceof SparkTask) {
+ SparkWork sparkWork = (SparkWork) currTask.getWork();
+ for (BaseWork baseWork : sparkWork.getAllWork()) {
+ if (baseWork instanceof MapWork) {
+ convertMapWork((MapWork) baseWork, false);
+ } else if (baseWork instanceof ReduceWork) {
+ convertReduceWork((ReduceWork) baseWork);
+ }
+ }
        }
        return null;
      }

Search Discussions

Discussion Posts

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 3 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedSep 25, '14 at 2:39p
activeSep 25, '14 at 2:39p
posts3
users1
websitehive.apache.org

1 user in discussion

Xuefu: 3 posts

People

Translate

site design / logo © 2021 Grokbase