Repository: hive Updated Branches: refs/heads/master 2218bd279 -> a7c197522
HIVE-17836 : Persisting nulls in bit vector field fails for postgres backed metastore (Ashutosh Chauhan via Zoltan Haindrich) Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a7c19752 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a7c19752 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a7c19752 Branch: refs/heads/master Commit: a7c197522ded9654996b1b47d28037cc51313b55 Parents: 2218bd2 Author: Ashutosh Chauhan <hashut...@apache.org> Authored: Thu Oct 19 08:52:30 2017 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Thu Oct 19 08:52:30 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/ObjectStore.java | 21 ++++++++++++++++---- .../hive/metastore/tools/SQLGenerator.java | 6 ++++++ 2 files changed, 23 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a7c19752/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index ffb2abd..af6a570 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -728,6 +728,7 @@ public class ObjectStore implements RawStore, Configurable { * @return true if there is an active transaction. If the current transaction * is either committed or rolled back it returns false */ + @Override public boolean isActiveTransaction() { if (currentTransaction == null) { return false; @@ -3254,13 +3255,14 @@ public class ObjectStore implements RawStore, Configurable { ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; return new GetHelper<Integer>(dbName, tblName, true, true) { - private SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @Override protected String describeResult() { return "Partition count"; } + @Override protected boolean canUseDirectSql(GetHelper<Integer> ctx) throws MetaException { return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter); } @@ -3285,13 +3287,14 @@ public class ObjectStore implements RawStore, Configurable { return new GetHelper<Integer>(dbName, tblName, true, true) { - private SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @Override protected String describeResult() { return "Partition count"; } + @Override protected boolean canUseDirectSql(GetHelper<Integer> ctx) throws MetaException { return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter); }; @@ -3331,7 +3334,7 @@ public class ObjectStore implements RawStore, Configurable { final ExpressionTree tree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) { - private SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @Override protected boolean canUseDirectSql(GetHelper<List<Partition>> ctx) throws MetaException { @@ -7280,6 +7283,11 @@ public class ObjectStore implements RawStore, Configurable { if (oldStats != null) { StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { + if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector() == null) { + // workaround for DN bug in persisting nulls in pg bytea column + // instead set empty bit vector with header. + mStatsObj.setBitVector(new byte[] {'H','L'}); + } pm.makePersistent(mStatsObj); } } finally { @@ -7316,6 +7324,11 @@ public class ObjectStore implements RawStore, Configurable { if (oldStats != null) { StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { + if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector() == null) { + // workaround for DN bug in persisting nulls in pg bytea column + // instead set empty bit vector with header. + mStatsObj.setBitVector(new byte[] {'H','L'}); + } pm.makePersistent(mStatsObj); } } finally { @@ -8801,7 +8814,7 @@ public class ObjectStore implements RawStore, Configurable { pmCache.setAccessible(true); Set<JDOPersistenceManager> pmSet = (Set<JDOPersistenceManager>)pmCache.get(pmf); for (JDOPersistenceManager pm : pmSet) { - org.datanucleus.ExecutionContext ec = (org.datanucleus.ExecutionContext)pm.getExecutionContext(); + org.datanucleus.ExecutionContext ec = pm.getExecutionContext(); if (ec instanceof org.datanucleus.ExecutionContextThreadedImpl) { ClassLoaderResolver clr = ((org.datanucleus.ExecutionContextThreadedImpl)ec).getClassLoaderResolver(); clearClr(clr); http://git-wip-us.apache.org/repos/asf/hive/blob/a7c19752/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java index 8268af9..5b4d4bd 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java @@ -38,6 +38,7 @@ import java.util.List; public final class SQLGenerator { static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName()); private final DatabaseProduct dbProduct; + private final Configuration conf; public SQLGenerator(DatabaseProduct dbProduct, Configuration conf) { @@ -169,4 +170,9 @@ public final class SQLGenerator { throw new MetaException(msg); } } + + public DatabaseProduct getDbProduct() { + return dbProduct; + } + }
