FAQ
Repository: hive
Updated Branches:
   refs/heads/master 6df52edc5 -> cfda5700a


HIVE-11376 : Removes legacy code wrt skipping files with codec to HiveInputFormat from CombineHiveInputFormat (Rajat Khandelwal, reviewed by Amareshwari)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cfda5700
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cfda5700
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cfda5700

Branch: refs/heads/master
Commit: cfda5700a715b71f5c4c6b325d4adaa213ff7618
Parents: 6df52ed
Author: Rajat Khandelwal <prongs@apache.org>
Authored: Mon Aug 10 17:51:09 2015 +0530
Committer: Amareshwari Sriramadasu <amareshwari@apache.org>
Committed: Mon Aug 10 17:51:09 2015 +0530

----------------------------------------------------------------------
  .../org/apache/hadoop/hive/conf/HiveConf.java | 2 -
  .../hive/ql/io/CombineHiveInputFormat.java | 39 --------------------
  .../hive/ql/optimizer/GenMapRedUtils.java | 2 -
  .../org/apache/hadoop/hive/ql/plan/MapWork.java | 10 -----
  .../apache/hadoop/hive/ql/plan/PlanUtils.java | 9 +----
  5 files changed, 1 insertion(+), 61 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 36bb394..9cc7987 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1175,8 +1175,6 @@ public class HiveConf extends Configuration {
      HIVEROWOFFSET("hive.exec.rowoffset", false,
          "Whether to provide the row offset virtual column"),

- HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false, ""),
-
      // Optimizer
      HIVEOPTINDEXFILTER("hive.optimize.index.filter", false,
          "Whether to enable automatic use of indexes"),

http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index e13c4dd..11740d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -374,45 +374,6 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
        }
        FileSystem inpFs = path.getFileSystem(job);

- // Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree or not,
- // we use a configuration variable for the same
- if (this.mrwork != null && !this.mrwork.getHadoopSupportsSplittable()) {
- // The following code should be removed, once
- // https://issues.apache.org/jira/browse/MAPREDUCE-1597 is fixed.
- // Hadoop does not handle non-splittable files correctly for CombineFileInputFormat,
- // so don't use CombineFileInputFormat for non-splittable files
-
- //ie, dont't combine if inputformat is a TextInputFormat and has compression turned on
-
- if (inputFormat instanceof TextInputFormat) {
- Queue<Path> dirs = new LinkedList<Path>();
- FileStatus fStats = inpFs.getFileStatus(path);
-
- // If path is a directory
- if (fStats.isDir()) {
- dirs.offer(path);
- } else if ((new CompressionCodecFactory(job)).getCodec(path) != null) {
- //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine)
- splits = super.getSplits(job, numSplits);
- return splits;
- }
-
- while (dirs.peek() != null) {
- Path tstPath = dirs.remove();
- FileStatus[] fStatus = inpFs.listStatus(tstPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
- for (int idx = 0; idx < fStatus.length; idx++) {
- if (fStatus[idx].isDir()) {
- dirs.offer(fStatus[idx].getPath());
- } else if ((new CompressionCodecFactory(job)).getCodec(
- fStatus[idx].getPath()) != null) {
- //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine)
- splits = super.getSplits(job, numSplits);
- return splits;
- }
- }
- }
- }
- }
        //don't combine if inputformat is a SymlinkTextInputFormat
        if (inputFormat instanceof SymlinkTextInputFormat) {
          splits = super.getSplits(job, numSplits);

http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 693d8c7..4a325fb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -933,8 +933,6 @@ public final class GenMapRedUtils {
      work.setPathToAliases(new LinkedHashMap<String, ArrayList<String>>());
      work.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>());
      work.setAliasToWork(new LinkedHashMap<String, Operator<? extends OperatorDesc>>());
- work.setHadoopSupportsSplittable(
- conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE));
      return mrWork;
    }


http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
index 2cb9257..bc9b645 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
@@ -63,8 +63,6 @@ public class MapWork extends BaseWork {

    private static final Log LOG = LogFactory.getLog(MapWork.class);

- private boolean hadoopSupportsSplittable;
-
    // use LinkedHashMap to make sure the iteration order is
    // deterministic, to ease testing
    private LinkedHashMap<String, ArrayList<String>> pathToAliases = new LinkedHashMap<String, ArrayList<String>>();
@@ -421,14 +419,6 @@ public class MapWork extends BaseWork {
      return this.mapperCannotSpanPartns;
    }

- public boolean getHadoopSupportsSplittable() {
- return hadoopSupportsSplittable;
- }
-
- public void setHadoopSupportsSplittable(boolean hadoopSupportsSplittable) {
- this.hadoopSupportsSplittable = hadoopSupportsSplittable;
- }
-
    public String getIndexIntermediateFile() {
      return indexIntermediateFile;
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 76926e7..b50eaab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -99,14 +99,7 @@ public final class PlanUtils {

    @SuppressWarnings("nls")
    public static MapredWork getMapRedWork() {
- try {
- MapredWork work = new MapredWork();
- work.getMapWork().setHadoopSupportsSplittable(Hive.get().getConf().getBoolVar(
- HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE));
- return work;
- } catch (HiveException ex) {
- throw new RuntimeException(ex);
- }
+ return new MapredWork();
    }

    public static TableDesc getDefaultTableDesc(CreateTableDesc directoryDesc,

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedAug 10, '15 at 12:21p
activeAug 10, '15 at 12:21p
posts1
users1
websitehive.apache.org

1 user in discussion

Amareshwari: 1 post

People

Translate

site design / logo © 2021 Grokbase