FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 f20a359d7 -> f953fc3cf


HIVE-11073 ORC FileDump utility ignores errors when writing output (backporting from 2.0.0)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f953fc3c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f953fc3c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f953fc3c

Branch: refs/heads/branch-1
Commit: f953fc3cf5625ed1002922aa69b7830e7e5f32d8
Parents: f20a359
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Wed Sep 30 18:23:17 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Wed Sep 30 18:23:17 2015 -0700

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/io/orc/FileDump.java | 7 ++-
  .../hadoop/hive/ql/io/orc/TestFileDump.java | 50 ++++++++++++++++++++
  2 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f953fc3c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
index 7b883cc..76ecb33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc;

  import java.io.IOException;
  import java.io.OutputStreamWriter;
+import java.io.PrintStream;
  import java.text.DecimalFormat;
  import java.util.ArrayList;
  import java.util.Collection;
@@ -481,7 +482,8 @@ public final class FileDump {
        String filename) throws IOException, JSONException {
      Path path = new Path(filename);
      Reader reader = OrcFile.createReader(path.getFileSystem(conf), path);
- OutputStreamWriter out = new OutputStreamWriter(System.out, "UTF-8");
+ PrintStream printStream = System.out;
+ OutputStreamWriter out = new OutputStreamWriter(printStream, "UTF-8");
      RecordReader rows = reader.rows(null);
      Object row = null;
      List<OrcProto.Type> types = reader.getTypes();
@@ -491,6 +493,9 @@ public final class FileDump {
        printObject(writer, row, types, 0);
        out.write("\n");
        out.flush();
+ if (printStream.checkError()) {
+ throw new IOException("Error encountered when writing to stdout.");
+ }
      }
    }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/f953fc3c/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
index 5b200f1..3d18d30 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
@@ -26,6 +26,8 @@ import java.io.ByteArrayOutputStream;
  import java.io.File;
  import java.io.FileOutputStream;
  import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStream;
  import java.io.PrintStream;
  import java.sql.Date;
  import java.sql.Timestamp;
@@ -247,6 +249,54 @@ public class TestFileDump {
      assertEquals("{\"b\":false,\"bt\":20,\"s\":200,\"i\":2000,\"l\":20000,\"f\":8,\"d\":40,\"de\":\"2.2222\",\"t\":\"2014-11-25 18:02:44\",\"dt\":\"2014-09-28\",\"str\":\"abcd\",\"c\":\"world \",\"vc\":\"world\",\"m\":[{\"_key\":\"k3\",\"_value\":\"v3\"}],\"a\":[200,300],\"st\":{\"i\":20,\"s\":\"bar\"}}", lines[1]);
    }

+ @Test(expected = IOException.class)
+ public void testDataDumpThrowsIOException() throws Exception {
+ PrintStream origOut = System.out;
+ try {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (AllTypesRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
+ 100000, CompressionKind.NONE, 10000, 1000);
+ Map<String, String> m = new HashMap<String, String>(2);
+ m.put("k1", "v1");
+ writer.addRow(new AllTypesRecord(
+ true,
+ (byte) 10,
+ (short) 100,
+ 1000,
+ 10000L,
+ 4.0f,
+ 20.0,
+ HiveDecimal.create("4.2222"),
+ new Timestamp(1416967764000L),
+ new Date(1416967764000L),
+ "string",
+ new HiveChar("hello", 5),
+ new HiveVarchar("hello", 10),
+ m,
+ Arrays.asList(100, 200),
+ new AllTypesRecord.Struct(10, "foo")));
+
+ writer.close();
+
+ OutputStream myOut = new OutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ throw new IOException();
+ }
+ };
+
+ // replace stdout and run command
+ System.setOut(new PrintStream(myOut));
+ FileDump.main(new String[]{testFilePath.toString(), "-d"});
+ } finally {
+ System.setOut(origOut);
+ }
+ }
+
    // Test that if the fraction of rows that have distinct strings is greater than the configured
    // threshold dictionary encoding is turned off. If dictionary encoding is turned off the length
    // of the dictionary stream for the column will be 0 in the ORC file dump.

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 1, '15 at 1:23a
activeOct 1, '15 at 1:23a
posts1
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 1 post

People

Translate

site design / logo © 2021 Grokbase