FAQ
Repository: hive
Updated Branches:
   refs/heads/master c0c19d073 -> 54ca6fc51


HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)
-- Add the forgotten new data file


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/54ca6fc5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/54ca6fc5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/54ca6fc5

Branch: refs/heads/master
Commit: 54ca6fc51239e28309dcc7d68647ee2a25f1ca98
Parents: c0c19d0
Author: Xuefu Zhang <xzhang@cloudera.com>
Authored: Mon Oct 5 05:46:20 2015 -0700
Committer: Xuefu Zhang <xzhang@cloudera.com>
Committed: Mon Oct 5 05:46:20 2015 -0700

----------------------------------------------------------------------
  data/files/decimal_1_1.txt | 30 ++++++++++++++++++++++++++++++
  1 file changed, 30 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/54ca6fc5/data/files/decimal_1_1.txt
----------------------------------------------------------------------
diff --git a/data/files/decimal_1_1.txt b/data/files/decimal_1_1.txt
new file mode 100644
index 0000000..ec16804
--- /dev/null
+++ b/data/files/decimal_1_1.txt
@@ -0,0 +1,30 @@
+0.0
+0.0000
+.0
+0.1
+0.15
+0.9
+0.94
+0.99
+0.345
+1.0
+1
+0
+00
+22
+1E-9
+-0.0
+-0.0000
+-.0
+-0.1
+-0.15
+-0.9
+-0.94
+-0.99
+-0.345
+-1.0
+-1
+-0
+-00
+-22
+-1E-9

Search Discussions

  • Xuefu at Oct 5, 2015 at 12:48 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 93b981f5d -> 7c0bf304b


    HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/01b55f4a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/01b55f4a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/01b55f4a

    Branch: refs/heads/branch-1
    Commit: 01b55f4aad72e3efb3f3165ab41deeeb81908453
    Parents: 93b981f
    Author: Xuefu Zhang <xzhang@cloudera.com>
    Authored: Mon Oct 5 05:42:20 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Oct 5 05:48:10 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/common/type/HiveDecimal.java | 10 +-
      .../hive/common/type/TestHiveDecimal.java | 12 +--
      .../test/resources/testconfiguration.properties | 1 +
      .../test/queries/clientpositive/decimal_1_1.q | 9 ++
      .../results/clientpositive/decimal_1_1.q.out | 104 +++++++++++++++++++
      .../clientpositive/spark/decimal_1_1.q.out | 104 +++++++++++++++++++
      6 files changed, 233 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
    index 5d2b4d2..da88b7a 100644
    --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
    +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
    @@ -272,11 +272,19 @@ public class HiveDecimal implements Comparable<HiveDecimal> {
          return bd;
        }

    - public static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
    + private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
          if (bd == null) {
            return null;
          }

    + /**
    + * Specially handling the case that bd=0, and we are converting it to a type where precision=scale,
    + * such as decimal(1, 1).
    + */
    + if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) {
    + return bd.setScale(maxScale);
    + }
    +
          bd = trim(bd);

          if (bd.scale() > maxScale) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
    ----------------------------------------------------------------------
    diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
    index ba5ef71..f68842c 100644
    --- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
    +++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
    @@ -42,15 +42,15 @@ public class TestHiveDecimal {
          Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE);

          decStr = "57847525803324040144343378.09799306448796128931113691624";
    - BigDecimal bd = new BigDecimal(decStr);
    - BigDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
    + HiveDecimal bd = HiveDecimal.create(decStr);
    + HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
          Assert.assertNull(bd1);
          bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5);
          Assert.assertEquals("57847525803324040144343378.09799", bd1.toString());
          bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20);
          Assert.assertNull(bd1);

    - dec = HiveDecimal.create(bd, false);
    + dec = HiveDecimal.create(new BigDecimal(decStr), false);
          Assert.assertNull(dec);

          dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624");
    @@ -65,10 +65,10 @@ public class TestHiveDecimal {

          // Rounding numbers that increase int digits
          Assert.assertEquals("10",
    - HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 2, 0).toString());
    - Assert.assertNull(HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 1, 0));
    + HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString());
    + Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0));
          Assert.assertEquals("9",
    - HiveDecimal.enforcePrecisionScale(new BigDecimal("9.4"), 1, 0).toString());
    + HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString());
        }

        @Test

    http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/itests/src/test/resources/testconfiguration.properties
    ----------------------------------------------------------------------
    diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
    index 0699441..faa9913 100644
    --- a/itests/src/test/resources/testconfiguration.properties
    +++ b/itests/src/test/resources/testconfiguration.properties
    @@ -653,6 +653,7 @@ spark.query.files=add_part_multiple.q, \
        custom_input_output_format.q, \
        date_join1.q, \
        date_udf.q, \
    + decimal_1_1.q, \
        decimal_join.q, \
        disable_merge_for_bucketing.q, \
        dynamic_rdd_cache.q, \

    http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/queries/clientpositive/decimal_1_1.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/decimal_1_1.q b/ql/src/test/queries/clientpositive/decimal_1_1.q
    new file mode 100644
    index 0000000..83ce521
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/decimal_1_1.q
    @@ -0,0 +1,9 @@
    +drop table if exists decimal_1_1;
    +
    +create table decimal_1_1 (d decimal(1,1));
    +load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1;
    +select * from decimal_1_1;
    +
    +select d from decimal_1_1 order by d desc;
    +
    +drop table decimal_1_1;

    http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/results/clientpositive/decimal_1_1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/decimal_1_1.q.out b/ql/src/test/results/clientpositive/decimal_1_1.q.out
    new file mode 100644
    index 0000000..b2704c6
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/decimal_1_1.q.out
    @@ -0,0 +1,104 @@
    +PREHOOK: query: drop table if exists decimal_1_1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table if exists decimal_1_1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@decimal_1_1
    +POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@decimal_1_1
    +PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@decimal_1_1
    +POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@decimal_1_1
    +PREHOOK: query: select * from decimal_1_1
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from decimal_1_1
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +0
    +0
    +0
    +0.1
    +0.2
    +0.9
    +0.9
    +NULL
    +0.3
    +NULL
    +NULL
    +0
    +0
    +NULL
    +0
    +0
    +0
    +0
    +-0.1
    +-0.2
    +-0.9
    +-0.9
    +NULL
    +-0.3
    +NULL
    +NULL
    +0
    +0
    +NULL
    +0
    +PREHOOK: query: select d from decimal_1_1 order by d desc
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select d from decimal_1_1 order by d desc
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +0.9
    +0.9
    +0.3
    +0.2
    +0.1
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +-0.1
    +-0.2
    +-0.3
    +-0.9
    +-0.9
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +PREHOOK: query: drop table decimal_1_1
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@decimal_1_1
    +PREHOOK: Output: default@decimal_1_1
    +POSTHOOK: query: drop table decimal_1_1
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@decimal_1_1
    +POSTHOOK: Output: default@decimal_1_1

    http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
    new file mode 100644
    index 0000000..b2704c6
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
    @@ -0,0 +1,104 @@
    +PREHOOK: query: drop table if exists decimal_1_1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table if exists decimal_1_1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@decimal_1_1
    +POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@decimal_1_1
    +PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@decimal_1_1
    +POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@decimal_1_1
    +PREHOOK: query: select * from decimal_1_1
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from decimal_1_1
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +0
    +0
    +0
    +0.1
    +0.2
    +0.9
    +0.9
    +NULL
    +0.3
    +NULL
    +NULL
    +0
    +0
    +NULL
    +0
    +0
    +0
    +0
    +-0.1
    +-0.2
    +-0.9
    +-0.9
    +NULL
    +-0.3
    +NULL
    +NULL
    +0
    +0
    +NULL
    +0
    +PREHOOK: query: select d from decimal_1_1 order by d desc
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select d from decimal_1_1 order by d desc
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@decimal_1_1
    +#### A masked pattern was here ####
    +0.9
    +0.9
    +0.3
    +0.2
    +0.1
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +0
    +-0.1
    +-0.2
    +-0.3
    +-0.9
    +-0.9
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +NULL
    +PREHOOK: query: drop table decimal_1_1
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@decimal_1_1
    +PREHOOK: Output: default@decimal_1_1
    +POSTHOOK: query: drop table decimal_1_1
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@decimal_1_1
    +POSTHOOK: Output: default@decimal_1_1
  • Xuefu at Oct 5, 2015 at 12:48 pm
    HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)
    -- Add the forgotten new data file


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c0bf304
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c0bf304
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c0bf304

    Branch: refs/heads/branch-1
    Commit: 7c0bf304bb61e32fb5b6bbfd3f4ed12e2f9a25b3
    Parents: 01b55f4
    Author: Xuefu Zhang <xzhang@cloudera.com>
    Authored: Mon Oct 5 05:46:20 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Mon Oct 5 05:48:11 2015 -0700

    ----------------------------------------------------------------------
      data/files/decimal_1_1.txt | 30 ++++++++++++++++++++++++++++++
      1 file changed, 30 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/7c0bf304/data/files/decimal_1_1.txt
    ----------------------------------------------------------------------
    diff --git a/data/files/decimal_1_1.txt b/data/files/decimal_1_1.txt
    new file mode 100644
    index 0000000..ec16804
    --- /dev/null
    +++ b/data/files/decimal_1_1.txt
    @@ -0,0 +1,30 @@
    +0.0
    +0.0000
    +.0
    +0.1
    +0.15
    +0.9
    +0.94
    +0.99
    +0.345
    +1.0
    +1
    +0
    +00
    +22
    +1E-9
    +-0.0
    +-0.0000
    +-.0
    +-0.1
    +-0.15
    +-0.9
    +-0.94
    +-0.99
    +-0.345
    +-1.0
    +-1
    +-0
    +-00
    +-22
    +-1E-9

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 5, '15 at 12:46p
activeOct 5, '15 at 12:48p
posts3
users1
websitehive.apache.org

1 user in discussion

Xuefu: 3 posts

People

Translate

site design / logo © 2021 Grokbase