FAQ
Author: xuefu
Date: Fri Dec 19 19:17:36 2014
New Revision: 1646833

URL: http://svn.apache.org/r1646833
Log:
HIVE-9088: Spark counter serialization error in spark.log [Spark Branch] (Chengxiang via Xuefu)

Modified:
     hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java?rev=1646833&r1=1646832&r2=1646833&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java Fri Dec 19 19:17:36 2014
@@ -376,14 +376,15 @@ public class SparkTask extends Task<Spar
      List<String> hiveCounters = new LinkedList<String>();
      counters.put(groupName, hiveCounters);
      hiveCounters.add(Operator.HIVECOUNTERCREATEDFILES);
+ // MapOperator is out of SparkWork, SparkMapRecordHandler use it to bridge
+ // Spark transformation and Hive operators in SparkWork.
+ for (MapOperator.Counter counter : MapOperator.Counter.values()) {
+ hiveCounters.add(counter.toString());
+ }
      SparkWork sparkWork = this.getWork();
      for (BaseWork work : sparkWork.getAllWork()) {
        for (Operator operator : work.getAllOperators()) {
- if (operator instanceof MapOperator) {
- for (MapOperator.Counter counter : MapOperator.Counter.values()) {
- hiveCounters.add(counter.toString());
- }
- } else if (operator instanceof FileSinkOperator) {
+ if (operator instanceof FileSinkOperator) {
            for (FileSinkOperator.Counter counter : FileSinkOperator.Counter.values()) {
              hiveCounters.add(counter.toString());
            }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 19, '14 at 7:17p
activeDec 19, '14 at 7:17p
posts1
users1
websitehive.apache.org

1 user in discussion

Xuefu: 1 post

People

Translate

site design / logo © 2021 Grokbase