hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-17836 : Persisting nulls in bit vector field fails for postgres backed metastore (Ashutosh Chauhan via Zoltan Haindrich)
Date Thu, 19 Oct 2017 15:53:13 GMT
Repository: hive
Updated Branches:
  refs/heads/master 2218bd279 -> a7c197522


HIVE-17836 : Persisting nulls in bit vector field fails for postgres backed metastore (Ashutosh
Chauhan via Zoltan Haindrich)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a7c19752
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a7c19752
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a7c19752

Branch: refs/heads/master
Commit: a7c197522ded9654996b1b47d28037cc51313b55
Parents: 2218bd2
Author: Ashutosh Chauhan <hashutosh@apache.org>
Authored: Thu Oct 19 08:52:30 2017 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Oct 19 08:52:30 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/metastore/ObjectStore.java      | 21 ++++++++++++++++----
 .../hive/metastore/tools/SQLGenerator.java      |  6 ++++++
 2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a7c19752/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index ffb2abd..af6a570 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -728,6 +728,7 @@ public class ObjectStore implements RawStore, Configurable {
    * @return true if there is an active transaction. If the current transaction
    *         is either committed or rolled back it returns false
    */
+  @Override
   public boolean isActiveTransaction() {
     if (currentTransaction == null) {
       return false;
@@ -3254,13 +3255,14 @@ public class ObjectStore implements RawStore, Configurable {
         ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
 
     return new GetHelper<Integer>(dbName, tblName, true, true) {
-      private SqlFilterForPushdown filter = new SqlFilterForPushdown();
+      private final SqlFilterForPushdown filter = new SqlFilterForPushdown();
 
       @Override
       protected String describeResult() {
         return "Partition count";
       }
 
+      @Override
       protected boolean canUseDirectSql(GetHelper<Integer> ctx) throws MetaException
{
         return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter);
       }
@@ -3285,13 +3287,14 @@ public class ObjectStore implements RawStore, Configurable {
 
 
     return new GetHelper<Integer>(dbName, tblName, true, true) {
-      private SqlFilterForPushdown filter = new SqlFilterForPushdown();
+      private final SqlFilterForPushdown filter = new SqlFilterForPushdown();
 
       @Override
       protected String describeResult() {
         return "Partition count";
       }
 
+      @Override
       protected boolean canUseDirectSql(GetHelper<Integer> ctx) throws MetaException
{
         return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter);
       };
@@ -3331,7 +3334,7 @@ public class ObjectStore implements RawStore, Configurable {
     final ExpressionTree tree = (filter != null && !filter.isEmpty())
         ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
     return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
-      private SqlFilterForPushdown filter = new SqlFilterForPushdown();
+      private final SqlFilterForPushdown filter = new SqlFilterForPushdown();
 
       @Override
       protected boolean canUseDirectSql(GetHelper<List<Partition>> ctx) throws
MetaException {
@@ -7280,6 +7283,11 @@ public class ObjectStore implements RawStore, Configurable {
       if (oldStats != null) {
         StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
       } else {
+        if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector()
== null) {
+          // workaround for DN bug in persisting nulls in pg bytea column
+          // instead set empty bit vector with header.
+          mStatsObj.setBitVector(new byte[] {'H','L'});
+        }
         pm.makePersistent(mStatsObj);
       }
     } finally {
@@ -7316,6 +7324,11 @@ public class ObjectStore implements RawStore, Configurable {
       if (oldStats != null) {
         StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats);
       } else {
+        if (sqlGenerator.getDbProduct().equals(DatabaseProduct.POSTGRES) && mStatsObj.getBitVector()
== null) {
+          // workaround for DN bug in persisting nulls in pg bytea column
+          // instead set empty bit vector with header.
+          mStatsObj.setBitVector(new byte[] {'H','L'});
+        }
         pm.makePersistent(mStatsObj);
       }
     } finally {
@@ -8801,7 +8814,7 @@ public class ObjectStore implements RawStore, Configurable {
       pmCache.setAccessible(true);
       Set<JDOPersistenceManager> pmSet = (Set<JDOPersistenceManager>)pmCache.get(pmf);
       for (JDOPersistenceManager pm : pmSet) {
-        org.datanucleus.ExecutionContext ec = (org.datanucleus.ExecutionContext)pm.getExecutionContext();
+        org.datanucleus.ExecutionContext ec = pm.getExecutionContext();
         if (ec instanceof org.datanucleus.ExecutionContextThreadedImpl) {
           ClassLoaderResolver clr = ((org.datanucleus.ExecutionContextThreadedImpl)ec).getClassLoaderResolver();
           clearClr(clr);

http://git-wip-us.apache.org/repos/asf/hive/blob/a7c19752/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
index 8268af9..5b4d4bd 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
@@ -38,6 +38,7 @@ import java.util.List;
 public final class SQLGenerator {
   static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName());
   private final DatabaseProduct dbProduct;
+
   private final Configuration conf;
 
   public SQLGenerator(DatabaseProduct dbProduct, Configuration conf) {
@@ -169,4 +170,9 @@ public final class SQLGenerator {
       throw new MetaException(msg);
     }
   }
+
+  public DatabaseProduct getDbProduct() {
+    return dbProduct;
+  }
+
 }


Mime
View raw message