hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject hive git commit: HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)
Date Mon, 27 Jul 2015 16:29:45 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 89f25ab0a -> ae827dba9


HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are
unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ae827dba
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ae827dba
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ae827dba

Branch: refs/heads/branch-1
Commit: ae827dba9cbaed136988d026e60b961d7cf56e04
Parents: 89f25ab
Author: Sushanth Sowmyan <khorgath@gmail.com>
Authored: Mon Jul 27 09:21:54 2015 -0700
Committer: Sushanth Sowmyan <khorgath@gmail.com>
Committed: Mon Jul 27 09:29:41 2015 -0700

----------------------------------------------------------------------
 .../hive/hcatalog/mapreduce/PartInfo.java       | 32 +++++++++++++++-----
 1 file changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ae827dba/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
index fca0a92..5b88505 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
@@ -173,7 +173,16 @@ public class PartInfo implements Serializable {
 
   void setTableInfo(HCatTableInfo thatTableInfo) {
     this.tableInfo = thatTableInfo;
+    restoreLocalInfoFromTableInfo();
+  }
 
+  /**
+   * Undoes the effects of compression( dedupWithTableInfo() ) during serialization,
+   * and restores PartInfo fields to return original data.
+   * Can be called idempotently, repeatably.
+   */
+  private void restoreLocalInfoFromTableInfo() {
+    assert tableInfo != null : "TableInfo can't be null at this point.";
     if (partitionSchema == null) {
       partitionSchema = tableInfo.getDataColumns();
     }
@@ -196,15 +205,10 @@ public class PartInfo implements Serializable {
   }
 
   /**
-   * Serialization method. Suppresses serialization of redundant information that's already
-   * available from TableInfo.
+   * Finds commonalities with TableInfo, and suppresses (nulls) fields if they are identical
    */
-  private void writeObject(ObjectOutputStream oos)
-      throws IOException {
-    // Suppress commonality with TableInfo.
-
+  private void dedupWithTableInfo() {
     assert tableInfo != null : "TableInfo can't be null at this point.";
-
     if (partitionSchema != null) {
       if (partitionSchema.equals(tableInfo.getDataColumns())) {
         partitionSchema = null;
@@ -260,7 +264,21 @@ public class PartInfo implements Serializable {
         }
       }
     }
+  }
 
+  /**
+   * Serialization method used by java serialization.
+   * Suppresses serialization of redundant information that's already available from
+   * TableInfo before writing out, so as to minimize amount of serialized space but
+   * restore it back before returning, so that PartInfo object is still usable afterwards
+   * (See HIVE-8485 and HIVE-11344 for details.)
+   */
+  private void writeObject(ObjectOutputStream oos)
+      throws IOException {
+    dedupWithTableInfo();
     oos.defaultWriteObject();
+    restoreLocalInfoFromTableInfo();
   }
+
+
 }


Mime
View raw message