FAQ
Repository: hive
Updated Branches:
   refs/heads/master 90a9a90ed -> db5fbe913


HIVE-13146: OrcFile table property values are case sensitive (Yongzhi Chen, reviewed by Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db5fbe91
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db5fbe91
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db5fbe91

Branch: refs/heads/master
Commit: db5fbe913eeff85e5b7ed238857b668f6449eb55
Parents: 90a9a90
Author: Yongzhi Chen <ychena@apache.org>
Authored: Thu Feb 25 12:54:03 2016 -0500
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Tue Mar 1 09:03:42 2016 -0500

----------------------------------------------------------------------
  orc/src/java/org/apache/orc/OrcFile.java | 2 +-
  .../hadoop/hive/ql/io/orc/ReaderImpl.java | 2 +-
  .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 1 +
  ql/src/test/queries/clientpositive/orc_create.q | 12 +++++
  .../results/clientpositive/orc_create.q.out | 52 ++++++++++++++++++++
  5 files changed, 67 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/orc/src/java/org/apache/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/OrcFile.java b/orc/src/java/org/apache/orc/OrcFile.java
index 98226f9..8061089 100644
--- a/orc/src/java/org/apache/orc/OrcFile.java
+++ b/orc/src/java/org/apache/orc/OrcFile.java
@@ -255,7 +255,7 @@ public class OrcFile {
            OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);
        compressValue =
            CompressionKind.valueOf(OrcConf.COMPRESS.getString(tableProperties,
- conf));
+ conf).toUpperCase());
        String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,
            conf);
        versionValue = Version.byName(versionName);

http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index 3e031ed..1299c9c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -585,7 +585,7 @@ public class ReaderImpl implements Reader {
      MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize,
          ByteBuffer footerBuffer) throws IOException {

- this.compressionKind = org.apache.orc.CompressionKind.valueOf(codecStr);
+ this.compressionKind = org.apache.orc.CompressionKind.valueOf(codecStr.toUpperCase());
        this.bufferSize = bufferSize;
        this.codec = WriterImpl.createCodec(compressionKind);
        this.metadataSize = metadataSize;

http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 0db1dab..6373746 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -12569,4 +12569,5 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
      SessionState.getConsole().printInfo(
          String.format("Warning: %s", msg));
    }
+
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/test/queries/clientpositive/orc_create.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_create.q b/ql/src/test/queries/clientpositive/orc_create.q
index 57912ef..0647200 100644
--- a/ql/src/test/queries/clientpositive/orc_create.q
+++ b/ql/src/test/queries/clientpositive/orc_create.q
@@ -6,6 +6,7 @@ DROP TABLE orc_create_complex;
  DROP TABLE orc_create_staging;
  DROP TABLE orc_create_people_staging;
  DROP TABLE orc_create_people;
+DROP TABLE if exists orc_create_cprl;

  CREATE TABLE orc_create_staging (
    str STRING,
@@ -119,8 +120,19 @@ SELECT COUNT(*) FROM orc_create_people where salary = 200.00 and state = 'Ca';
  SELECT id, first_name, last_name, address
    FROM orc_create_people WHERE id > 90;

+-- test create with lower case compression method.
+CREATE TABLE orc_create_cprl (id int)
+PARTITIONED BY (cdate date)
+STORED AS ORC
+TBLPROPERTIES (
+'orc.compress'='snappy');
+INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03')
+SELECT 1 from src limit 1;
+SELECT * from orc_create_cprl;
+
  DROP TABLE orc_create;
  DROP TABLE orc_create_complex;
  DROP TABLE orc_create_staging;
  DROP TABLE orc_create_people_staging;
  DROP TABLE orc_create_people;
+DROP TABLE orc_create_cprl;

http://git-wip-us.apache.org/repos/asf/hive/blob/db5fbe91/ql/src/test/results/clientpositive/orc_create.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_create.q.out b/ql/src/test/results/clientpositive/orc_create.q.out
index e294dec..3013fda 100644
--- a/ql/src/test/results/clientpositive/orc_create.q.out
+++ b/ql/src/test/results/clientpositive/orc_create.q.out
@@ -22,6 +22,10 @@ PREHOOK: query: DROP TABLE orc_create_people
  PREHOOK: type: DROPTABLE
  POSTHOOK: query: DROP TABLE orc_create_people
  POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE if exists orc_create_cprl
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE if exists orc_create_cprl
+POSTHOOK: type: DROPTABLE
  PREHOOK: query: CREATE TABLE orc_create_staging (
    str STRING,
    mp MAP<STRING,STRING>,
@@ -678,6 +682,46 @@ POSTHOOK: Input: default@orc_create_people@state=Or
  97 Dana Carter 814-601 Purus. Av.
  98 Juliet Battle Ap #535-1965 Cursus St.
  99 Wynter Vincent 626-8492 Mollis Avenue
+PREHOOK: query: -- test create with lower case compression method.
+CREATE TABLE orc_create_cprl (id int)
+PARTITIONED BY (cdate date)
+STORED AS ORC
+TBLPROPERTIES (
+'orc.compress'='snappy')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_create_cprl
+POSTHOOK: query: -- test create with lower case compression method.
+CREATE TABLE orc_create_cprl (id int)
+PARTITIONED BY (cdate date)
+STORED AS ORC
+TBLPROPERTIES (
+'orc.compress'='snappy')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_create_cprl
+PREHOOK: query: INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03')
+SELECT 1 from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@orc_create_cprl@cdate=2015-02-03
+POSTHOOK: query: INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03')
+SELECT 1 from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@orc_create_cprl@cdate=2015-02-03
+POSTHOOK: Lineage: orc_create_cprl PARTITION(cdate=2015-02-03).id SIMPLE []
+PREHOOK: query: SELECT * from orc_create_cprl
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_cprl
+PREHOOK: Input: default@orc_create_cprl@cdate=2015-02-03
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_cprl
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_cprl
+POSTHOOK: Input: default@orc_create_cprl@cdate=2015-02-03
+#### A masked pattern was here ####
+1 2015-02-03
  PREHOOK: query: DROP TABLE orc_create
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@orc_create
@@ -718,3 +762,11 @@ POSTHOOK: query: DROP TABLE orc_create_people
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@orc_create_people
  POSTHOOK: Output: default@orc_create_people
+PREHOOK: query: DROP TABLE orc_create_cprl
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_cprl
+PREHOOK: Output: default@orc_create_cprl
+POSTHOOK: query: DROP TABLE orc_create_cprl
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_cprl
+POSTHOOK: Output: default@orc_create_cprl

Search Discussions

  • Ychena at Mar 1, 2016 at 2:46 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 f1c75b5f0 -> 82c2acdbe


    HIVE-13146: OrcFile table property values are case sensitive (Yongzhi Chen, reviewed by Aihua Xu)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82c2acdb
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82c2acdb
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82c2acdb

    Branch: refs/heads/branch-1
    Commit: 82c2acdbe10cf7a27d58cf0f1fb291b7cdb18e54
    Parents: f1c75b5
    Author: Yongzhi Chen <ychena@apache.org>
    Authored: Thu Feb 25 12:54:03 2016 -0500
    Committer: Yongzhi Chen <ychena@apache.org>
    Committed: Tue Mar 1 09:45:07 2016 -0500

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 2 +-
      .../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 2 +-
      .../hadoop/hive/ql/io/orc/ReaderImpl.java | 2 +-
      ql/src/test/queries/clientpositive/orc_create.q | 12 +++++
      .../results/clientpositive/orc_create.q.out | 52 ++++++++++++++++++++
      5 files changed, 67 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
    index 08f2f5e..b13dad2 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
    @@ -307,7 +307,7 @@ public final class OrcFile {
            propValue = tableProperties == null ? null
                : tableProperties.getProperty(OrcTableProperties.COMPRESSION.propName);
            compressValue = propValue == null ? CompressionKind.valueOf(HiveConf.getVar(conf, HIVE_ORC_DEFAULT_COMPRESS))
    - : CompressionKind.valueOf(propValue);
    + : CompressionKind.valueOf(propValue.toUpperCase());

            propValue = tableProperties == null ? null
                : tableProperties.getProperty(OrcTableProperties.BLOOM_FILTER_COLUMNS.propName);

    http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
    index 8625ff1..bc55677 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
    @@ -140,7 +140,7 @@ public class OrcOutputFormat extends FileOutputFormat<NullWritable, OrcSerdeRow>

          if ((propVal = getSettingFromPropsFallingBackToConf(
              OrcFile.OrcTableProperties.COMPRESSION.getPropName(),props,conf)) != null){
    - options.compress(CompressionKind.valueOf(propVal));
    + options.compress(CompressionKind.valueOf(propVal.toUpperCase()));
          }

          if ((propVal = getSettingFromPropsFallingBackToConf(

    http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    index fc1a8fa..a7c564a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    @@ -456,7 +456,7 @@ public class ReaderImpl implements Reader {
          MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize,
              ByteBuffer footerBuffer) throws IOException {

    - this.compressionKind = CompressionKind.valueOf(codecStr);
    + this.compressionKind = CompressionKind.valueOf(codecStr.toUpperCase());
            this.bufferSize = bufferSize;
            this.codec = WriterImpl.createCodec(compressionKind);
            this.metadataSize = metadataSize;

    http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/test/queries/clientpositive/orc_create.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/orc_create.q b/ql/src/test/queries/clientpositive/orc_create.q
    index e42c5ae..dc28105 100644
    --- a/ql/src/test/queries/clientpositive/orc_create.q
    +++ b/ql/src/test/queries/clientpositive/orc_create.q
    @@ -5,6 +5,7 @@ DROP TABLE orc_create_complex;
      DROP TABLE orc_create_staging;
      DROP TABLE orc_create_people_staging;
      DROP TABLE orc_create_people;
    +DROP TABLE if exists orc_create_cprl;

      CREATE TABLE orc_create_staging (
        str STRING,
    @@ -118,8 +119,19 @@ SELECT COUNT(*) FROM orc_create_people where salary = 200.00 and state = 'Ca';
      SELECT id, first_name, last_name, address
        FROM orc_create_people WHERE id > 90;

    +-- test create with lower case compression method.
    +CREATE TABLE orc_create_cprl (id int)
    +PARTITIONED BY (cdate date)
    +STORED AS ORC
    +TBLPROPERTIES (
    +'orc.compress'='snappy');
    +INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03')
    +SELECT 1 from src limit 1;
    +SELECT * from orc_create_cprl;
    +
      DROP TABLE orc_create;
      DROP TABLE orc_create_complex;
      DROP TABLE orc_create_staging;
      DROP TABLE orc_create_people_staging;
      DROP TABLE orc_create_people;
    +DROP TABLE orc_create_cprl;

    http://git-wip-us.apache.org/repos/asf/hive/blob/82c2acdb/ql/src/test/results/clientpositive/orc_create.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/orc_create.q.out b/ql/src/test/results/clientpositive/orc_create.q.out
    index bffb58d..8153e98 100644
    --- a/ql/src/test/results/clientpositive/orc_create.q.out
    +++ b/ql/src/test/results/clientpositive/orc_create.q.out
    @@ -22,6 +22,10 @@ PREHOOK: query: DROP TABLE orc_create_people
      PREHOOK: type: DROPTABLE
      POSTHOOK: query: DROP TABLE orc_create_people
      POSTHOOK: type: DROPTABLE
    +PREHOOK: query: DROP TABLE if exists orc_create_cprl
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE if exists orc_create_cprl
    +POSTHOOK: type: DROPTABLE
      PREHOOK: query: CREATE TABLE orc_create_staging (
        str STRING,
        mp MAP<STRING,STRING>,
    @@ -684,6 +688,46 @@ POSTHOOK: Input: default@orc_create_people@state=Or
      97 Dana Carter 814-601 Purus. Av.
      98 Juliet Battle Ap #535-1965 Cursus St.
      99 Wynter Vincent 626-8492 Mollis Avenue
    +PREHOOK: query: -- test create with lower case compression method.
    +CREATE TABLE orc_create_cprl (id int)
    +PARTITIONED BY (cdate date)
    +STORED AS ORC
    +TBLPROPERTIES (
    +'orc.compress'='snappy')
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@orc_create_cprl
    +POSTHOOK: query: -- test create with lower case compression method.
    +CREATE TABLE orc_create_cprl (id int)
    +PARTITIONED BY (cdate date)
    +STORED AS ORC
    +TBLPROPERTIES (
    +'orc.compress'='snappy')
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@orc_create_cprl
    +PREHOOK: query: INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03')
    +SELECT 1 from src limit 1
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@orc_create_cprl@cdate=2015-02-03
    +POSTHOOK: query: INSERT OVERWRITE table orc_create_cprl PARTITION (cdate = '2015-02-03')
    +SELECT 1 from src limit 1
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@orc_create_cprl@cdate=2015-02-03
    +POSTHOOK: Lineage: orc_create_cprl PARTITION(cdate=2015-02-03).id SIMPLE []
    +PREHOOK: query: SELECT * from orc_create_cprl
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@orc_create_cprl
    +PREHOOK: Input: default@orc_create_cprl@cdate=2015-02-03
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * from orc_create_cprl
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@orc_create_cprl
    +POSTHOOK: Input: default@orc_create_cprl@cdate=2015-02-03
    +#### A masked pattern was here ####
    +1 2015-02-03
      PREHOOK: query: DROP TABLE orc_create
      PREHOOK: type: DROPTABLE
      PREHOOK: Input: default@orc_create
    @@ -724,3 +768,11 @@ POSTHOOK: query: DROP TABLE orc_create_people
      POSTHOOK: type: DROPTABLE
      POSTHOOK: Input: default@orc_create_people
      POSTHOOK: Output: default@orc_create_people
    +PREHOOK: query: DROP TABLE orc_create_cprl
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@orc_create_cprl
    +PREHOOK: Output: default@orc_create_cprl
    +POSTHOOK: query: DROP TABLE orc_create_cprl
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@orc_create_cprl
    +POSTHOOK: Output: default@orc_create_cprl

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 1, '16 at 2:04p
activeMar 1, '16 at 2:46p
posts2
users1
websitehive.apache.org

1 user in discussion

Ychena: 2 posts

People

Translate

site design / logo © 2021 Grokbase