FAQ
Author: hashutosh
Date: Sat Mar 24 01:08:02 2012
New Revision: 1304693

URL: http://svn.apache.org/viewvc?rev=1304693&view=rev
Log:
HIVE-2894 [jira] RCFile Reader doesn't provide access to Metadata
(Owen O'Malley via Ashutosh Chauhan)

Summary:
hive-2894

Add an accessor for RCFile's metadata.

Currently the RCFile writer can add metadata to an RCFile, but the reader
doesn't provide an accessor. I'd like to add one.

Test Plan:
I added a call to test that the metadata that was passed in was available from
the reader.

Reviewers: JIRA, ashutoshc

Reviewed By: ashutoshc

Differential Revision: https://reviews.facebook.net/D2421

Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1304693&r1=1304692&r2=1304693&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Sat Mar 24 01:08:02 2012
@@ -1187,6 +1187,14 @@ public class RCFile {
}

/**
+ * Return the metadata (string to string map) that was written into the
+ * file.
+ */
+ public Metadata getMetadata() {
+ return metadata;
+ }
+
+ /**
* Override this method to specialize the type of
* {@link FSDataInputStream} returned.
*/

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=1304693&r1=1304692&r2=1304693&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Sat Mar 24 01:08:02 2012
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -148,10 +149,11 @@ public class TestRCFile extends TestCase
"123".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
"5.3".getBytes("UTF-8"), "hive and hadoop".getBytes("UTF-8"),
new byte[0], "NULL".getBytes("UTF-8")};
-
+ Metadata metadata = new Metadata();
+ metadata.set(new Text("apple"), new Text("block"));
RCFileOutputFormat.setColumnNumber(conf, expectedFieldsData.length);
RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
- new DefaultCodec());
+ metadata, new DefaultCodec());
BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
for (int i = 0; i < record_1.length; i++) {
BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
@@ -179,7 +181,8 @@ public class TestRCFile extends TestCase
new Text("hive and hadoop"), null, null};

RCFile.Reader reader = new RCFile.Reader(fs, file, conf);
-
+ assertEquals(new Text("block"),
+ reader.getMetadata().get(new Text("apple")));
LongWritable rowID = new LongWritable();

for (int i = 0; i < 2; i++) {
@@ -518,10 +521,10 @@ public class TestRCFile extends TestCase
}
assertEquals("readCount should be equal to writeCount", readCount, writeCount);
}
-
+

// adopted Hadoop-5476 (calling new SequenceFile.Reader(...) leaves an
- // InputStream open, if the given sequence file is broken) to RCFile
+ // InputStream open, if the given sequence file is broken) to RCFile
private static class TestFSDataInputStream extends FSDataInputStream {
private boolean closed = false;

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 24, '12 at 1:08a
activeMar 24, '12 at 1:08a
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase