FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 93b981f5d -> 7c0bf304b


HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/01b55f4a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/01b55f4a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/01b55f4a

Branch: refs/heads/branch-1
Commit: 01b55f4aad72e3efb3f3165ab41deeeb81908453
Parents: 93b981f
Author: Xuefu Zhang <xzhang@cloudera.com>
Authored: Mon Oct 5 05:42:20 2015 -0700
Committer: Xuefu Zhang <xzhang@cloudera.com>
Committed: Mon Oct 5 05:48:10 2015 -0700

----------------------------------------------------------------------
  .../hadoop/hive/common/type/HiveDecimal.java | 10 +-
  .../hive/common/type/TestHiveDecimal.java | 12 +--
  .../test/resources/testconfiguration.properties | 1 +
  .../test/queries/clientpositive/decimal_1_1.q | 9 ++
  .../results/clientpositive/decimal_1_1.q.out | 104 +++++++++++++++++++
  .../clientpositive/spark/decimal_1_1.q.out | 104 +++++++++++++++++++
  6 files changed, 233 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
index 5d2b4d2..da88b7a 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
@@ -272,11 +272,19 @@ public class HiveDecimal implements Comparable<HiveDecimal> {
      return bd;
    }

- public static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
+ private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
      if (bd == null) {
        return null;
      }

+ /**
+ * Specially handling the case that bd=0, and we are converting it to a type where precision=scale,
+ * such as decimal(1, 1).
+ */
+ if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) {
+ return bd.setScale(maxScale);
+ }
+
      bd = trim(bd);

      if (bd.scale() > maxScale) {

http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
index ba5ef71..f68842c 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
@@ -42,15 +42,15 @@ public class TestHiveDecimal {
      Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE);

      decStr = "57847525803324040144343378.09799306448796128931113691624";
- BigDecimal bd = new BigDecimal(decStr);
- BigDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
+ HiveDecimal bd = HiveDecimal.create(decStr);
+ HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
      Assert.assertNull(bd1);
      bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5);
      Assert.assertEquals("57847525803324040144343378.09799", bd1.toString());
      bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20);
      Assert.assertNull(bd1);

- dec = HiveDecimal.create(bd, false);
+ dec = HiveDecimal.create(new BigDecimal(decStr), false);
      Assert.assertNull(dec);

      dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624");
@@ -65,10 +65,10 @@ public class TestHiveDecimal {

      // Rounding numbers that increase int digits
      Assert.assertEquals("10",
- HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 2, 0).toString());
- Assert.assertNull(HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 1, 0));
+ HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString());
+ Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0));
      Assert.assertEquals("9",
- HiveDecimal.enforcePrecisionScale(new BigDecimal("9.4"), 1, 0).toString());
+ HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString());
    }

    @Test

http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 0699441..faa9913 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -653,6 +653,7 @@ spark.query.files=add_part_multiple.q, \
    custom_input_output_format.q, \
    date_join1.q, \
    date_udf.q, \
+ decimal_1_1.q, \
    decimal_join.q, \
    disable_merge_for_bucketing.q, \
    dynamic_rdd_cache.q, \

http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/queries/clientpositive/decimal_1_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/decimal_1_1.q b/ql/src/test/queries/clientpositive/decimal_1_1.q
new file mode 100644
index 0000000..83ce521
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/decimal_1_1.q
@@ -0,0 +1,9 @@
+drop table if exists decimal_1_1;
+
+create table decimal_1_1 (d decimal(1,1));
+load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1;
+select * from decimal_1_1;
+
+select d from decimal_1_1 order by d desc;
+
+drop table decimal_1_1;

http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/results/clientpositive/decimal_1_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_1_1.q.out b/ql/src/test/results/clientpositive/decimal_1_1.q.out
new file mode 100644
index 0000000..b2704c6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/decimal_1_1.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: drop table if exists decimal_1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: select * from decimal_1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from decimal_1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0
+0
+0
+0.1
+0.2
+0.9
+0.9
+NULL
+0.3
+NULL
+NULL
+0
+0
+NULL
+0
+0
+0
+0
+-0.1
+-0.2
+-0.9
+-0.9
+NULL
+-0.3
+NULL
+NULL
+0
+0
+NULL
+0
+PREHOOK: query: select d from decimal_1_1 order by d desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select d from decimal_1_1 order by d desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0.9
+0.9
+0.3
+0.2
+0.1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+-0.1
+-0.2
+-0.3
+-0.9
+-0.9
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: drop table decimal_1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_1_1
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: drop table decimal_1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_1_1
+POSTHOOK: Output: default@decimal_1_1

http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
new file mode 100644
index 0000000..b2704c6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: drop table if exists decimal_1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: select * from decimal_1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from decimal_1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0
+0
+0
+0.1
+0.2
+0.9
+0.9
+NULL
+0.3
+NULL
+NULL
+0
+0
+NULL
+0
+0
+0
+0
+-0.1
+-0.2
+-0.9
+-0.9
+NULL
+-0.3
+NULL
+NULL
+0
+0
+NULL
+0
+PREHOOK: query: select d from decimal_1_1 order by d desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select d from decimal_1_1 order by d desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0.9
+0.9
+0.3
+0.2
+0.1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+-0.1
+-0.2
+-0.3
+-0.9
+-0.9
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: drop table decimal_1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_1_1
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: drop table decimal_1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_1_1
+POSTHOOK: Output: default@decimal_1_1

Search Discussions

Discussion Posts

Previous

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 3 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 5, '15 at 12:46p
activeOct 5, '15 at 12:48p
posts3
users1
websitehive.apache.org

1 user in discussion

Xuefu: 3 posts

People

Translate

site design / logo © 2021 Grokbase