hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-20765 : fetch partitions for txn stats validation in get_aggr_stats with one call (Sergey Shelukhin, reviewed by Ashutosh Chauhan, and Vihang Karajgaonkar)
Date Mon, 22 Oct 2018 21:07:07 GMT
Repository: hive
Updated Branches:
  refs/heads/master b4302bb7a -> cbe3228c2


HIVE-20765 : fetch partitions for txn stats validation in get_aggr_stats with one call (Sergey
Shelukhin, reviewed by Ashutosh Chauhan, and Vihang Karajgaonkar)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbe3228c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbe3228c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbe3228c

Branch: refs/heads/master
Commit: cbe3228c29e2db6ae5cffcb94cdcf79063859ffe
Parents: b4302bb
Author: sergey <sershe@apache.org>
Authored: Mon Oct 22 14:06:48 2018 -0700
Committer: sergey <sershe@apache.org>
Committed: Mon Oct 22 14:06:48 2018 -0700

----------------------------------------------------------------------
 .../hadoop/hive/metastore/ObjectStore.java      | 22 +++++++++++++-------
 .../metastore/PartitionProjectionEvaluator.java |  8 +++++--
 2 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b98b4b4..ddd64e7 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -9052,21 +9052,27 @@ public class ObjectStore implements RawStore, Configurable {
         return null;
       }
 
-      MTable table = getMTable(catName, dbName, tblName);
+      Table table = getTable(catName, dbName, tblName);
       boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters());
       if (isTxn && !areTxnStatsSupported) {
         return null;
       }
+      GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec();
+      fs.setFilterMode(PartitionFilterMode.BY_NAMES);
+      fs.setFilters(partNames);
+      GetPartitionsProjectionSpec ps = new GetPartitionsProjectionSpec();
+      ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%');
+      ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values"));
+      List<Partition> parts = getPartitionSpecsByFilterAndProjection(table, ps, fs);
 
       // Loop through the given "partNames" list
       // checking isolation-level-compliance of each partition column stats.
-      for (String partName : partNames) {
-        MPartition mpart = getMPartition(
-            catName, dbName, tblName, Warehouse.getPartValuesFromPartName(partName));
-        if (!isCurrentStatsValidForTheQuery(mpart, writeIdList, false)) {
-          LOG.debug("The current metastore transactional partition column statistics " +
-                  "for " + dbName + "." + tblName + "." + mpart.getPartitionName() + " is
not valid " +
-                  "for the current query.");
+      for (Partition part : parts) {
+
+        if (!isCurrentStatsValidForTheQuery(part, part.getWriteId(), writeIdList, false))
{
+          String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+          LOG.debug("The current metastore transactional partition column statistics for
" + dbName
+              + "." + tblName + "." + partName + " is not valid for the current query");
           return null;
         }
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
index e918a33..c3a0093 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
@@ -26,6 +26,7 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Partition;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.jdo.PersistenceManager;
 import javax.jdo.Query;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -369,8 +371,8 @@ public class PartitionProjectionEvaluator {
     int numColumns = buildQueryForSingleValuedFields(partitionIds, queryTextBuilder);
     String queryText = queryTextBuilder.toString();
 
-    try (Query query = pm.newQuery("javax.jdo.query.SQL", queryText)) {
-
+    Query<?> query = pm.newQuery("javax.jdo.query.SQL", queryText);
+    try {
       long start = LOG.isDebugEnabled() ? System.nanoTime() : 0;
       List<Object> sqlResult = MetastoreDirectSqlUtils.executeWithArray(query, null,
queryText);
       long queryTime = LOG.isDebugEnabled() ? System.nanoTime() : 0;
@@ -456,6 +458,8 @@ public class PartitionProjectionEvaluator {
     } catch (Exception e) {
       LOG.error("Exception received while getting partitions using projected fields", e);
       throw new MetaException(e.getMessage());
+    } finally {
+      query.closeAll();
     }
   }
 


Mime
View raw message