FAQ
Author: gunther
Date: Sat Mar 7 01:55:52 2015
New Revision: 1664780

URL: http://svn.apache.org/r1664780
Log:
HIVE-9891: LLAP: disable plan caching (Gunther Hagleitner)

Modified:
     hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
     hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java?rev=1664780&r1=1664779&r2=1664780&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java Sat Mar 7 01:55:52 2015
@@ -25,13 +25,13 @@ import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
-import java.util.TreeMap;
  import java.util.concurrent.Callable;

  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
  import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
  import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
  import org.apache.hadoop.hive.ql.exec.MapOperator;
@@ -84,7 +84,11 @@ public class MapRecordProcessor extends
    ObjectCache cache;

    public MapRecordProcessor(final JobConf jconf) throws Exception {
- cache = ObjectCacheFactory.getCache(jconf);
+ if (LlapIoProxy.isDaemon()) { // do not cache plan
+ cache = new org.apache.hadoop.hive.ql.exec.mr.ObjectCache();
+ } else {
+ cache = ObjectCacheFactory.getCache(jconf);
+ }
      execContext = new ExecMapperContext(jconf);
      execContext.setJc(jconf);
      cacheKeys = new ArrayList<String>();

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1664780&r1=1664779&r2=1664780&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java Sat Mar 7 01:55:52 2015
@@ -27,6 +27,7 @@ import java.util.concurrent.Callable;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
  import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
  import org.apache.hadoop.hive.ql.exec.MapredContext;
  import org.apache.hadoop.hive.ql.exec.ObjectCache;
@@ -80,7 +81,13 @@ public class ReduceRecordProcessor exte
      perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS);
      super.init(jconf, processorContext, mrReporter, inputs, outputs);

- ObjectCache cache = ObjectCacheFactory.getCache(jconf);
+ ObjectCache cache;
+
+ if (LlapIoProxy.isDaemon()) { // don't cache plan
+ cache = new org.apache.hadoop.hive.ql.exec.mr.ObjectCache();
+ } else {
+ cache = ObjectCacheFactory.getCache(jconf);
+ }

      String queryId = HiveConf.getVar(jconf, HiveConf.ConfVars.HIVEQUERYID);
      cacheKey = queryId + processorContext.getTaskVertexName() + REDUCE_PLAN_KEY;

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 7, '15 at 1:55a
activeMar 7, '15 at 1:55a
posts1
users1
websitehive.apache.org

1 user in discussion

Gunther: 1 post

People

Translate

site design / logo © 2021 Grokbase