FAQ
Repository: hive
Updated Branches:
   refs/heads/master 92e98858e -> 9f474263b


HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f474263
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f474263
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f474263

Branch: refs/heads/master
Commit: 9f474263b5c47dc25c096a0b54794ebcda146bed
Parents: 92e9885
Author: Sushanth Sowmyan <khorgath@gmail.com>
Authored: Mon Jul 27 09:21:54 2015 -0700
Committer: Sushanth Sowmyan <khorgath@gmail.com>
Committed: Mon Jul 27 09:29:18 2015 -0700

----------------------------------------------------------------------
  .../hive/hcatalog/mapreduce/PartInfo.java | 32 +++++++++++++++-----
  1 file changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9f474263/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
index fca0a92..5b88505 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
@@ -173,7 +173,16 @@ public class PartInfo implements Serializable {

    void setTableInfo(HCatTableInfo thatTableInfo) {
      this.tableInfo = thatTableInfo;
+ restoreLocalInfoFromTableInfo();
+ }

+ /**
+ * Undoes the effects of compression( dedupWithTableInfo() ) during serialization,
+ * and restores PartInfo fields to return original data.
+ * Can be called idempotently, repeatably.
+ */
+ private void restoreLocalInfoFromTableInfo() {
+ assert tableInfo != null : "TableInfo can't be null at this point.";
      if (partitionSchema == null) {
        partitionSchema = tableInfo.getDataColumns();
      }
@@ -196,15 +205,10 @@ public class PartInfo implements Serializable {
    }

    /**
- * Serialization method. Suppresses serialization of redundant information that's already
- * available from TableInfo.
+ * Finds commonalities with TableInfo, and suppresses (nulls) fields if they are identical
     */
- private void writeObject(ObjectOutputStream oos)
- throws IOException {
- // Suppress commonality with TableInfo.
-
+ private void dedupWithTableInfo() {
      assert tableInfo != null : "TableInfo can't be null at this point.";
-
      if (partitionSchema != null) {
        if (partitionSchema.equals(tableInfo.getDataColumns())) {
          partitionSchema = null;
@@ -260,7 +264,21 @@ public class PartInfo implements Serializable {
          }
        }
      }
+ }

+ /**
+ * Serialization method used by java serialization.
+ * Suppresses serialization of redundant information that's already available from
+ * TableInfo before writing out, so as to minimize amount of serialized space but
+ * restore it back before returning, so that PartInfo object is still usable afterwards
+ * (See HIVE-8485 and HIVE-11344 for details.)
+ */
+ private void writeObject(ObjectOutputStream oos)
+ throws IOException {
+ dedupWithTableInfo();
      oos.defaultWriteObject();
+ restoreLocalInfoFromTableInfo();
    }
+
+
  }

Search Discussions

  • Khorgath at Jul 27, 2015 at 4:30 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 89f25ab0a -> ae827dba9


    HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ae827dba
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ae827dba
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ae827dba

    Branch: refs/heads/branch-1
    Commit: ae827dba9cbaed136988d026e60b961d7cf56e04
    Parents: 89f25ab
    Author: Sushanth Sowmyan <khorgath@gmail.com>
    Authored: Mon Jul 27 09:21:54 2015 -0700
    Committer: Sushanth Sowmyan <khorgath@gmail.com>
    Committed: Mon Jul 27 09:29:41 2015 -0700

    ----------------------------------------------------------------------
      .../hive/hcatalog/mapreduce/PartInfo.java | 32 +++++++++++++++-----
      1 file changed, 25 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/ae827dba/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    index fca0a92..5b88505 100644
    --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    @@ -173,7 +173,16 @@ public class PartInfo implements Serializable {

        void setTableInfo(HCatTableInfo thatTableInfo) {
          this.tableInfo = thatTableInfo;
    + restoreLocalInfoFromTableInfo();
    + }

    + /**
    + * Undoes the effects of compression( dedupWithTableInfo() ) during serialization,
    + * and restores PartInfo fields to return original data.
    + * Can be called idempotently, repeatably.
    + */
    + private void restoreLocalInfoFromTableInfo() {
    + assert tableInfo != null : "TableInfo can't be null at this point.";
          if (partitionSchema == null) {
            partitionSchema = tableInfo.getDataColumns();
          }
    @@ -196,15 +205,10 @@ public class PartInfo implements Serializable {
        }

        /**
    - * Serialization method. Suppresses serialization of redundant information that's already
    - * available from TableInfo.
    + * Finds commonalities with TableInfo, and suppresses (nulls) fields if they are identical
         */
    - private void writeObject(ObjectOutputStream oos)
    - throws IOException {
    - // Suppress commonality with TableInfo.
    -
    + private void dedupWithTableInfo() {
          assert tableInfo != null : "TableInfo can't be null at this point.";
    -
          if (partitionSchema != null) {
            if (partitionSchema.equals(tableInfo.getDataColumns())) {
              partitionSchema = null;
    @@ -260,7 +264,21 @@ public class PartInfo implements Serializable {
              }
            }
          }
    + }

    + /**
    + * Serialization method used by java serialization.
    + * Suppresses serialization of redundant information that's already available from
    + * TableInfo before writing out, so as to minimize amount of serialized space but
    + * restore it back before returning, so that PartInfo object is still usable afterwards
    + * (See HIVE-8485 and HIVE-11344 for details.)
    + */
    + private void writeObject(ObjectOutputStream oos)
    + throws IOException {
    + dedupWithTableInfo();
          oos.defaultWriteObject();
    + restoreLocalInfoFromTableInfo();
        }
    +
    +
      }
  • Khorgath at Jul 27, 2015 at 4:30 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1.2 d43f36d81 -> 4ccc0c37a


    HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ccc0c37
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ccc0c37
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ccc0c37

    Branch: refs/heads/branch-1.2
    Commit: 4ccc0c37aabbd90ecaa36fcc491e2270e7e9bea6
    Parents: d43f36d
    Author: Sushanth Sowmyan <khorgath@gmail.com>
    Authored: Mon Jul 27 09:21:54 2015 -0700
    Committer: Sushanth Sowmyan <khorgath@gmail.com>
    Committed: Mon Jul 27 09:29:58 2015 -0700

    ----------------------------------------------------------------------
      .../hive/hcatalog/mapreduce/PartInfo.java | 32 +++++++++++++++-----
      1 file changed, 25 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/4ccc0c37/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    index fca0a92..5b88505 100644
    --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
    @@ -173,7 +173,16 @@ public class PartInfo implements Serializable {

        void setTableInfo(HCatTableInfo thatTableInfo) {
          this.tableInfo = thatTableInfo;
    + restoreLocalInfoFromTableInfo();
    + }

    + /**
    + * Undoes the effects of compression( dedupWithTableInfo() ) during serialization,
    + * and restores PartInfo fields to return original data.
    + * Can be called idempotently, repeatably.
    + */
    + private void restoreLocalInfoFromTableInfo() {
    + assert tableInfo != null : "TableInfo can't be null at this point.";
          if (partitionSchema == null) {
            partitionSchema = tableInfo.getDataColumns();
          }
    @@ -196,15 +205,10 @@ public class PartInfo implements Serializable {
        }

        /**
    - * Serialization method. Suppresses serialization of redundant information that's already
    - * available from TableInfo.
    + * Finds commonalities with TableInfo, and suppresses (nulls) fields if they are identical
         */
    - private void writeObject(ObjectOutputStream oos)
    - throws IOException {
    - // Suppress commonality with TableInfo.
    -
    + private void dedupWithTableInfo() {
          assert tableInfo != null : "TableInfo can't be null at this point.";
    -
          if (partitionSchema != null) {
            if (partitionSchema.equals(tableInfo.getDataColumns())) {
              partitionSchema = null;
    @@ -260,7 +264,21 @@ public class PartInfo implements Serializable {
              }
            }
          }
    + }

    + /**
    + * Serialization method used by java serialization.
    + * Suppresses serialization of redundant information that's already available from
    + * TableInfo before writing out, so as to minimize amount of serialized space but
    + * restore it back before returning, so that PartInfo object is still usable afterwards
    + * (See HIVE-8485 and HIVE-11344 for details.)
    + */
    + private void writeObject(ObjectOutputStream oos)
    + throws IOException {
    + dedupWithTableInfo();
          oos.defaultWriteObject();
    + restoreLocalInfoFromTableInfo();
        }
    +
    +
      }

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 27, '15 at 4:29p
activeJul 27, '15 at 4:30p
posts3
users1
websitehive.apache.org

1 user in discussion

Khorgath: 3 posts

People

Translate

site design / logo © 2021 Grokbase