impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mi...@apache.org
Subject [1/4] incubator-impala git commit: IMPALA-5954: Set DO_NOT_UPDATE_STATS in alterTable() RPCs to HMS.
Date Fri, 22 Sep 2017 17:13:22 GMT
Repository: incubator-impala
Updated Branches:
  refs/heads/master 537ae013e -> 495325440


IMPALA-5954: Set DO_NOT_UPDATE_STATS in alterTable() RPCs to HMS.

This improves an existing workaround for HIVE-12730 to avoid
having the HMS wipe stats as a side-effect of an alterTable()
RPC. The new workaround uses DO_NOT_UPDATE_STATS which is
the recommended way of telling the HMS not to recompute/reset
statistics on its own.

Testing:
- core/hdfs run passed

Change-Id: I05c63315579d66efeaa4c65105372ae68820ab2f
Reviewed-on: http://gerrit.cloudera.org:8080/8112
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/eb276d48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/eb276d48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/eb276d48

Branch: refs/heads/master
Commit: eb276d48e4fc651d910ccb3af78a235eac190905
Parents: 537ae01
Author: Alex Behm <alex.behm@cloudera.com>
Authored: Mon Sep 18 18:41:10 2017 -0700
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Fri Sep 22 00:53:49 2017 +0000

----------------------------------------------------------------------
 .../org/apache/impala/service/CatalogOpExecutor.java | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eb276d48/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index e791de1..cf1e677 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -797,6 +797,7 @@ public class CatalogOpExecutor {
       }
       PartitionStatsUtil.partStatsToParameters(partitionStats, partition);
       partition.putToParameters(StatsSetupConst.ROW_COUNT, String.valueOf(numRows));
+      // HMS requires this param for stats changes to take effect.
       partition.putToParameters(MetastoreShim.statsGeneratedViaStatsTaskParam());
       ++numTargetedPartitions;
       modifiedParts.add(partition);
@@ -819,6 +820,7 @@ public class CatalogOpExecutor {
       msTbl.putToParameters(StatsSetupConst.RAW_DATA_SIZE,
           String.valueOf(params.getTable_stats().total_file_bytes));
     }
+    // HMS requires this param for stats changes to take effect.
     Pair<String, String> statsTaskParam = MetastoreShim.statsGeneratedViaStatsTaskParam();
     msTbl.putToParameters(statsTaskParam.first, statsTaskParam.second);
     return numTargetedPartitions;
@@ -2729,7 +2731,8 @@ public class CatalogOpExecutor {
     }
   }
 
-  /** Applies an ALTER TABLE command to the metastore table.
+  /**
+   * Applies an ALTER TABLE command to the metastore table.
    * Note: The metastore interface is not very safe because it only accepts
    * an entire metastore.api.Table object rather than a delta of what to change. This
    * means an external modification to the table could be overwritten by an ALTER TABLE
@@ -2743,11 +2746,11 @@ public class CatalogOpExecutor {
     try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
       lastDdlTime = calculateDdlTime(msTbl);
       msTbl.putToParameters("transient_lastDdlTime", Long.toString(lastDdlTime));
-      // TODO: Remove this workaround for HIVE-15653 to preserve table stats
-      // during table alterations.
-      Pair<String, String> statsTaskParam =
-          MetastoreShim.statsGeneratedViaStatsTaskParam();
-      msTbl.putToParameters(statsTaskParam.first, statsTaskParam.second);
+      // Avoid computing/setting stats on the HMS side because that may reset the
+      // 'numRows' table property (see HIVE-15653). The DO_NOT_UPDATE_STATS flag
+      // tells the HMS not to recompute/reset any statistics on its own. Any
+      // stats-related alterations passed in the RPC will still be applied.
+      msTbl.putToParameters(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
       msClient.getHiveClient().alter_table(
           msTbl.getDbName(), msTbl.getTableName(), msTbl);
     } catch (TException e) {


Mime
View raw message