HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f474263 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f474263 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f474263 Branch: refs/heads/spark Commit: 9f474263b5c47dc25c096a0b54794ebcda146bed Parents: 92e9885 Author: Sushanth Sowmyan <khorg...@gmail.com> Authored: Mon Jul 27 09:21:54 2015 -0700 Committer: Sushanth Sowmyan <khorg...@gmail.com> Committed: Mon Jul 27 09:29:18 2015 -0700 ---------------------------------------------------------------------- .../hive/hcatalog/mapreduce/PartInfo.java | 32 +++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9f474263/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java index fca0a92..5b88505 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java @@ -173,7 +173,16 @@ public class PartInfo implements Serializable { void setTableInfo(HCatTableInfo thatTableInfo) { this.tableInfo = thatTableInfo; + restoreLocalInfoFromTableInfo(); + } + /** + * Undoes the effects of compression( dedupWithTableInfo() ) during serialization, + * and restores PartInfo fields to return original data. + * Can be called idempotently, repeatably. + */ + private void restoreLocalInfoFromTableInfo() { + assert tableInfo != null : "TableInfo can't be null at this point."; if (partitionSchema == null) { partitionSchema = tableInfo.getDataColumns(); } @@ -196,15 +205,10 @@ public class PartInfo implements Serializable { } /** - * Serialization method. Suppresses serialization of redundant information that's already - * available from TableInfo. + * Finds commonalities with TableInfo, and suppresses (nulls) fields if they are identical */ - private void writeObject(ObjectOutputStream oos) - throws IOException { - // Suppress commonality with TableInfo. - + private void dedupWithTableInfo() { assert tableInfo != null : "TableInfo can't be null at this point."; - if (partitionSchema != null) { if (partitionSchema.equals(tableInfo.getDataColumns())) { partitionSchema = null; @@ -260,7 +264,21 @@ public class PartInfo implements Serializable { } } } + } + /** + * Serialization method used by java serialization. + * Suppresses serialization of redundant information that's already available from + * TableInfo before writing out, so as to minimize amount of serialized space but + * restore it back before returning, so that PartInfo object is still usable afterwards + * (See HIVE-8485 and HIVE-11344 for details.) + */ + private void writeObject(ObjectOutputStream oos) + throws IOException { + dedupWithTableInfo(); oos.defaultWriteObject(); + restoreLocalInfoFromTableInfo(); } + + }