FAQ
Author: hashutosh
Date: Tue Nov 27 17:49:05 2012
New Revision: 1414286

URL: http://svn.apache.org/viewvc?rev=1414286&view=rev
Log:
HIVE-3645 : RCFileWriter does not implement the right function to support Federation (Arup Malakar via Ashutosh Chauhan)

Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1414286&r1=1414285&r2=1414286&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Tue Nov 27 17:49:05 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.serde2.Col
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.hive.serde2.columnar.LazyDecompressionCallback;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile.Metadata;
@@ -732,7 +733,8 @@ public class RCFile {
public Writer(FileSystem fs, Configuration conf, Path name,
Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException {
this(fs, conf, name, fs.getConf().getInt("io.file.buffer.size", 4096),
- fs.getDefaultReplication(), fs.getDefaultBlockSize(), progress,
+ ShimLoader.getHadoopShims().getDefaultReplication(fs, name),
+ ShimLoader.getHadoopShims().getDefaultBlockSize(fs, name), progress,
metadata, codec);
}


Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1414286&r1=1414285&r2=1414286&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Tue Nov 27 17:49:05 2012
@@ -624,4 +624,14 @@ public class Hadoop20Shims implements Ha
public String getJobLauncherHttpAddress(Configuration conf) {
return conf.get("mapred.job.tracker.http.address");
}
+
+ @Override
+ public long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize();
+ }
+
+ @Override
+ public short getDefaultReplication(FileSystem fs, Path path) {
+ return fs.getDefaultReplication();
+ }
}

Modified: hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1414286&r1=1414285&r2=1414286&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Tue Nov 27 17:49:05 2012
@@ -21,6 +21,8 @@ import java.net.MalformedURLException;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
@@ -93,4 +95,14 @@ public class Hadoop20SShims extends Hado
public String getJobLauncherHttpAddress(Configuration conf) {
return conf.get("mapred.job.tracker.http.address");
}
+
+ @Override
+ public long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize();
+ }
+
+ @Override
+ public short getDefaultReplication(FileSystem fs, Path path) {
+ return fs.getDefaultReplication();
+ }
}

Modified: hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1414286&r1=1414285&r2=1414286&view=diff
==============================================================================
--- hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Tue Nov 27 17:49:05 2012
@@ -22,6 +22,8 @@ import java.net.MalformedURLException;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
import org.apache.hadoop.mapred.ClusterStatus;
@@ -113,4 +115,15 @@ public class Hadoop23Shims extends Hadoo
return conf.get("yarn.resourcemanager.webapp.address");
}

+ @Override
+ public long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize(path);
+ }
+
+ @Override
+ public short getDefaultReplication(FileSystem fs, Path path) {
+ return fs.getDefaultReplication(path);
+ }
+
+
}

Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1414286&r1=1414285&r2=1414286&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Tue Nov 27 17:49:05 2012
@@ -576,4 +576,10 @@ public abstract class HadoopShimsSecure

@Override
abstract public String getJobLauncherRpcAddress(Configuration conf);
+
+ @Override
+ abstract public short getDefaultReplication(FileSystem fs, Path path);
+
+ @Override
+ abstract public long getDefaultBlockSize(FileSystem fs, Path path);
}

Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1414286&r1=1414285&r2=1414286&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Tue Nov 27 17:49:05 2012
@@ -290,6 +290,24 @@ public interface HadoopShims {
public String getJobLauncherHttpAddress(Configuration conf);

/**
+ * Get the default block size for the path. FileSystem alone is not sufficient to
+ * determine the same, as in case of CSMT the underlying file system determines that.
+ * @param fs
+ * @param path
+ * @return
+ */
+ public long getDefaultBlockSize(FileSystem fs, Path path);
+
+ /**
+ * Get the default replication for a path. In case of CSMT the given path will be used to
+ * locate the actual filesystem.
+ * @param fs
+ * @param path
+ * @return
+ */
+ public short getDefaultReplication(FileSystem fs, Path path);
+
+ /**
* InputSplitShim.
*
*/

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 27, '12 at 5:49p
activeNov 27, '12 at 5:49p
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase