FAQ
Author: gunther
Date: Fri Nov 15 22:51:31 2013
New Revision: 1542424

URL: http://svn.apache.org/r1542424
Log:
HIVE-5834: Avoid reading ORC footers for files which will not be split in OrcInputFormat::getSplits() (Gopal V via Gunther Hagleitner)

Modified:
     hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1542424&r1=1542423&r2=1542424&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Fri Nov 15 22:51:31 2013
@@ -475,7 +475,8 @@ public class OrcInputFormat implements
              if (context.cacheStripeDetails) {
                fileInfo = verifyCachedFileInfo(file);
              }
- context.schedule(new SplitGenerator(context, fs, file, fileInfo));
+ SplitGenerator spgen = new SplitGenerator(context, fs, file, fileInfo);
+ spgen.schedule();
            }
          }
        } catch (Throwable th) {
@@ -553,6 +554,19 @@ public class OrcInputFormat implements
        return file.getPath();
      }

+ void schedule() throws IOException {
+ if(locations.length == 1 && file.getLen() < context.maxSize) {
+ String[] hosts = locations[0].getHosts();
+ synchronized (context.splits) {
+ context.splits.add(new OrcSplit(file.getPath(), 0, file.getLen(),
+ hosts, fileMetaInfo));
+ }
+ } else {
+ // if it requires a compute task
+ context.schedule(this);
+ }
+ }
+
      @Override
      public String toString() {
        return "splitter(" + file.getPath() + ")";
@@ -877,4 +891,4 @@ public class OrcInputFormat implements
        this.types = types;
      }
    }
-}
\ No newline at end of file
+}

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 15, '13 at 10:51p
activeNov 15, '13 at 10:51p
posts1
users1
websitehive.apache.org

1 user in discussion

Gunther: 1 post

People

Translate

site design / logo © 2021 Grokbase