FAQ
Repository: hive
Updated Branches:
   refs/heads/master 368f33165 -> 8b0f753c9


HIVE-11073 : ORC FileDump utility ignores errors when writing output (Elliot West via Alan Gates)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8b0f753c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8b0f753c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8b0f753c

Branch: refs/heads/master
Commit: 8b0f753c9c63f0a913d4bf3bb2e9b2a7873fe7fc
Parents: 368f331
Author: Elliot West <teabot@gmail.com>
Authored: Mon Jun 22 10:00:00 2015 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Sat Jun 27 15:02:24 2015 -0700

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/io/orc/FileDump.java | 7 ++-
  .../hadoop/hive/ql/io/orc/TestFileDump.java | 50 ++++++++++++++++++++
  2 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8b0f753c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
index 33c4cd8..cbbec36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc;

  import java.io.IOException;
  import java.io.OutputStreamWriter;
+import java.io.PrintStream;
  import java.text.DecimalFormat;
  import java.util.ArrayList;
  import java.util.Arrays;
@@ -437,7 +438,8 @@ public final class FileDump {
        String filename) throws IOException, JSONException {
      Path path = new Path(filename);
      Reader reader = OrcFile.createReader(path.getFileSystem(conf), path);
- OutputStreamWriter out = new OutputStreamWriter(System.out, "UTF-8");
+ PrintStream printStream = System.out;
+ OutputStreamWriter out = new OutputStreamWriter(printStream, "UTF-8");
      RecordReader rows = reader.rows(null);
      Object row = null;
      List<OrcProto.Type> types = reader.getTypes();
@@ -447,6 +449,9 @@ public final class FileDump {
        printObject(writer, row, types, 0);
        out.write("\n");
        out.flush();
+ if (printStream.checkError()) {
+ throw new IOException("Error encountered when writing to stdout.");
+ }
      }
    }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/8b0f753c/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
index e77e24d..68d503e 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
@@ -26,6 +26,8 @@ import java.io.ByteArrayOutputStream;
  import java.io.File;
  import java.io.FileOutputStream;
  import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStream;
  import java.io.PrintStream;
  import java.sql.Date;
  import java.sql.Timestamp;
@@ -249,6 +251,54 @@ public class TestFileDump {
      assertEquals("{\"b\":true,\"bt\":10,\"s\":100,\"i\":1000,\"l\":10000,\"f\":4,\"d\":20,\"de\":\"4.2222\",\"t\":\"2014-11-25 18:09:24\",\"dt\":\"2014-11-25\",\"str\":\"string\",\"c\":\"hello \",\"vc\":\"hello\",\"m\":[{\"_key\":\"k1\",\"_value\":\"v1\"}],\"a\":[100,200],\"st\":{\"i\":10,\"s\":\"foo\"}}", lines[0]);
      assertEquals("{\"b\":false,\"bt\":20,\"s\":200,\"i\":2000,\"l\":20000,\"f\":8,\"d\":40,\"de\":\"2.2222\",\"t\":\"2014-11-25 18:02:44\",\"dt\":\"2014-09-28\",\"str\":\"abcd\",\"c\":\"world \",\"vc\":\"world\",\"m\":[{\"_key\":\"k3\",\"_value\":\"v3\"}],\"a\":[200,300],\"st\":{\"i\":20,\"s\":\"bar\"}}", lines[1]);
    }
+
+ @Test(expected = IOException.class)
+ public void testDataDumpThrowsIOException() throws Exception {
+ PrintStream origOut = System.out;
+ try {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (AllTypesRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
+ 100000, CompressionKind.NONE, 10000, 1000);
+ Map<String, String> m = new HashMap<String, String>(2);
+ m.put("k1", "v1");
+ writer.addRow(new AllTypesRecord(
+ true,
+ (byte) 10,
+ (short) 100,
+ 1000,
+ 10000L,
+ 4.0f,
+ 20.0,
+ HiveDecimal.create("4.2222"),
+ new Timestamp(1416967764000L),
+ new Date(1416967764000L),
+ "string",
+ new HiveChar("hello", 5),
+ new HiveVarchar("hello", 10),
+ m,
+ Arrays.asList(100, 200),
+ new AllTypesRecord.Struct(10, "foo")));
+
+ writer.close();
+
+ OutputStream myOut = new OutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ throw new IOException();
+ }
+ };
+
+ // replace stdout and run command
+ System.setOut(new PrintStream(myOut));
+ FileDump.main(new String[]{testFilePath.toString(), "-d"});
+ } finally {
+ System.setOut(origOut);
+ }
+ }

    // Test that if the fraction of rows that have distinct strings is greater than the configured
    // threshold dictionary encoding is turned off. If dictionary encoding is turned off the length

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 27, '15 at 10:02p
activeJun 27, '15 at 10:02p
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase