FAQ
Repository: hive
Updated Branches:
   refs/heads/master de260b45d -> d4c1fdcf2


HIVE-13251: hive can't read the decimal in AVRO file generated from previous version (Reviewed by Szehon Ho)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d4c1fdcf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d4c1fdcf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d4c1fdcf

Branch: refs/heads/master
Commit: d4c1fdcf26ad8a12d77841def6afead86377b1ac
Parents: de260b4
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Thu Mar 10 10:29:40 2016 -0500
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Mar 14 11:07:24 2016 -0400

----------------------------------------------------------------------
  data/files/dec_old.avro | Bin 0 -> 331 bytes
  .../test/queries/clientnegative/avro_decimal.q | 17 ++++++
  .../queries/clientpositive/avro_decimal_old.q | 14 +++++
  .../results/clientnegative/avro_decimal.q.out | 22 +++++++
  .../clientpositive/avro_decimal_old.q.out | 60 +++++++++++++++++++
  .../hive/serde2/avro/AvroDeserializer.java | 2 +-
  6 files changed, 114 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/data/files/dec_old.avro
----------------------------------------------------------------------
diff --git a/data/files/dec_old.avro b/data/files/dec_old.avro
new file mode 100644
index 0000000..bf87763
Binary files /dev/null and b/data/files/dec_old.avro differ

http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/queries/clientnegative/avro_decimal.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/avro_decimal.q b/ql/src/test/queries/clientnegative/avro_decimal.q
new file mode 100644
index 0000000..538e687
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/avro_decimal.q
@@ -0,0 +1,17 @@
+DROP TABLE IF EXISTS avro_dec;
+
+CREATE TABLE `avro_dec`(
+ `name` string COMMENT 'from deserializer',
+ `value` decimal(5,2) COMMENT 'from deserializer')
+COMMENT 'just drop the schema right into the HQL'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+TBLPROPERTIES (
+ 'numFiles'='1',
+ 'avro.schema.literal'='{\"namespace\":\"com.howdy\",\"name\":\"some_schema\",\"type\":\"record\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":"5",\"scale\":"2"}}]}'
+);
+

http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/queries/clientpositive/avro_decimal_old.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/avro_decimal_old.q b/ql/src/test/queries/clientpositive/avro_decimal_old.q
new file mode 100644
index 0000000..9610c47
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/avro_decimal_old.q
@@ -0,0 +1,14 @@
+DROP TABLE IF EXISTS avro_dec_old;
+
+CREATE TABLE `avro_dec_old`(
+ `name` string COMMENT 'from deserializer',
+ `value` decimal(4,1) COMMENT 'from deserializer')
+STORED AS AVRO;
+
+DESC avro_dec_old;
+
+LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old;
+
+select value from avro_dec_old;
+
+DROP TABLE avro_dec_old;

http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/results/clientnegative/avro_decimal.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/avro_decimal.q.out b/ql/src/test/results/clientnegative/avro_decimal.q.out
new file mode 100644
index 0000000..9d00d6e
--- /dev/null
+++ b/ql/src/test/results/clientnegative/avro_decimal.q.out
@@ -0,0 +1,22 @@
+PREHOOK: query: DROP TABLE IF EXISTS avro_dec
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS avro_dec
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE `avro_dec`(
+ `name` string COMMENT 'from deserializer',
+ `value` decimal(5,2) COMMENT 'from deserializer')
+COMMENT 'just drop the schema right into the HQL'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+TBLPROPERTIES (
+ 'numFiles'='1',
+ 'avro.schema.literal'='{\"namespace\":\"com.howdy\",\"name\":\"some_schema\",\"type\":\"record\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":"5",\"scale\":"2"}}]}'
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@avro_dec
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.avro.AvroSerdeException Invalid precision or scale for decimal type)

http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/results/clientpositive/avro_decimal_old.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/avro_decimal_old.q.out b/ql/src/test/results/clientpositive/avro_decimal_old.q.out
new file mode 100644
index 0000000..22efe39
--- /dev/null
+++ b/ql/src/test/results/clientpositive/avro_decimal_old.q.out
@@ -0,0 +1,60 @@
+PREHOOK: query: DROP TABLE IF EXISTS avro_dec_old
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS avro_dec_old
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE `avro_dec_old`(
+ `name` string COMMENT 'from deserializer',
+ `value` decimal(4,1) COMMENT 'from deserializer')
+STORED AS AVRO
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@avro_dec_old
+POSTHOOK: query: CREATE TABLE `avro_dec_old`(
+ `name` string COMMENT 'from deserializer',
+ `value` decimal(4,1) COMMENT 'from deserializer')
+STORED AS AVRO
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@avro_dec_old
+PREHOOK: query: DESC avro_dec_old
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@avro_dec_old
+POSTHOOK: query: DESC avro_dec_old
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@avro_dec_old
+name string from deserializer
+value decimal(4,1) from deserializer
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@avro_dec_old
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@avro_dec_old
+PREHOOK: query: select value from avro_dec_old
+PREHOOK: type: QUERY
+PREHOOK: Input: default@avro_dec_old
+#### A masked pattern was here ####
+POSTHOOK: query: select value from avro_dec_old
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@avro_dec_old
+#### A masked pattern was here ####
+234.8
+77.3
+55.7
+4.3
+6.0
+12.3
+33.3
+19.0
+3.2
+79.9
+PREHOOK: query: DROP TABLE avro_dec_old
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@avro_dec_old
+PREHOOK: Output: default@avro_dec_old
+POSTHOOK: query: DROP TABLE avro_dec_old
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@avro_dec_old
+POSTHOOK: Output: default@avro_dec_old

http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
index 4bba3d4..6165138 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
@@ -244,7 +244,7 @@ class AvroDeserializer {

        int scale = 0;
        try {
- scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue();
+ scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).asInt();
        } catch(Exception ex) {
          throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex);
        }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 14, '16 at 3:08p
activeMar 14, '16 at 3:08p
posts1
users1
websitehive.apache.org

1 user in discussion

Aihuaxu: 1 post

People

Translate

site design / logo © 2021 Grokbase