FAQ
Author: xuefu
Date: Fri Dec 19 19:19:29 2014
New Revision: 1646835

URL: http://svn.apache.org/r1646835
Log:
HIVE-9154: Cache pathToPartitionInfo in context aware record reader (Chao via Xuefu)

Modified:
     hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java?rev=1646835&r1=1646834&r2=1646835&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java Fri Dec 19 19:19:29 2014
@@ -29,8 +29,8 @@ import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
-import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.exec.FooterBuffer;
+import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities;
  import org.apache.hadoop.hive.ql.io.IOContext.Comparison;
  import org.apache.hadoop.hive.ql.plan.PartitionDesc;
@@ -68,6 +68,7 @@ public abstract class HiveContextAwareRe
    private boolean wasUsingSortedSearch = false;
    private String genericUDFClassName = null;
    private final List<Comparison> stopComparisons = new ArrayList<Comparison>();
+ private Map<String, PartitionDesc> pathToPartitionInfo;

    protected RecordReader recordReader;
    protected JobConf jobConf;
@@ -185,6 +186,8 @@ public abstract class HiveContextAwareRe
        IOContext.getMap().put(SparkUtilities.MAP_IO_CONTEXT, iocontext);
      }

+ pathToPartitionInfo =
+ Utilities.getMapWork(jobConf).getPathToPartitionInfo();
      initDone = true;
    }

@@ -324,8 +327,6 @@ public abstract class HiveContextAwareRe
          Path filePath = this.ioCxtRef.getInputPath();
          PartitionDesc part = null;
          try {
- Map<String, PartitionDesc> pathToPartitionInfo = Utilities
- .getMapWork(jobConf).getPathToPartitionInfo();
            part = HiveFileFormatUtils
                .getPartitionDescFromPathRecursively(pathToPartitionInfo,
                    filePath, IOPrepareCache.get().getPartitionDescMap());

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 19, '14 at 7:19p
activeDec 19, '14 at 7:19p
posts1
users1
websitehive.apache.org

1 user in discussion

Xuefu: 1 post

People

Translate

site design / logo © 2021 Grokbase