FAQ
Author: khorgath
Date: Wed Nov 19 00:31:20 2014
New Revision: 1640458

URL: http://svn.apache.org/r1640458
Log:
HIVE-8808 : HiveInputFormat caching cannot work with all input formats (Brock Noland via Sushanth Sowmyan)

Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1640458&r1=1640457&r2=1640458&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Wed Nov 19 00:31:20 2014
@@ -197,18 +197,22 @@ public class HiveInputFormat<K extends W

    public static InputFormat<WritableComparable, Writable> getInputFormatFromCache(
      Class inputFormatClass, JobConf job) throws IOException {
-
- if (!inputFormats.containsKey(inputFormatClass)) {
+ InputFormat<WritableComparable, Writable> instance = inputFormats.get(inputFormatClass);
+ if (instance == null) {
        try {
- InputFormat<WritableComparable, Writable> newInstance = (InputFormat<WritableComparable, Writable>) ReflectionUtils
+ instance = (InputFormat<WritableComparable, Writable>) ReflectionUtils
              .newInstance(inputFormatClass, job);
- inputFormats.put(inputFormatClass, newInstance);
+ // HBase input formats are not thread safe today. See HIVE-8808.
+ String inputFormatName = inputFormatClass.getName().toLowerCase();
+ if (!inputFormatName.contains("hbase")) {
+ inputFormats.put(inputFormatClass, instance);
+ }
        } catch (Exception e) {
          throw new IOException("Cannot create an instance of InputFormat class "
              + inputFormatClass.getName() + " as specified in mapredWork!", e);
        }
      }
- return inputFormats.get(inputFormatClass);
+ return instance;
    }

    public RecordReader getRecordReader(InputSplit split, JobConf job,

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 19, '14 at 12:33a
activeNov 19, '14 at 12:33a
posts1
users1
websitehive.apache.org

1 user in discussion

Khorgath: 1 post

People

Translate

site design / logo © 2021 Grokbase