Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 84B3F18089 for ; Mon, 24 Aug 2015 17:15:40 +0000 (UTC) Received: (qmail 86129 invoked by uid 500); 24 Aug 2015 17:15:40 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 86079 invoked by uid 500); 24 Aug 2015 17:15:40 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 86066 invoked by uid 99); 24 Aug 2015 17:15:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 24 Aug 2015 17:15:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 327CBE09FB; Mon, 24 Aug 2015 17:15:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hashutosh@apache.org To: commits@hive.apache.org Message-Id: <315e381ee8124c9fa057295d39d372b4@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-10631 : create_table_core method has invalid update for Fast Stats (Aaron Tokhy via Ashutosh Chauhan) Date: Mon, 24 Aug 2015 17:15:40 +0000 (UTC) Repository: hive Updated Branches: refs/heads/branch-1 92bd50eed -> f4aac7ef4 HIVE-10631 : create_table_core method has invalid update for Fast Stats (Aaron Tokhy via Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4aac7ef Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4aac7ef Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4aac7ef Branch: refs/heads/branch-1 Commit: f4aac7ef4c34c2947ca5caa4cf9cf6e09f920771 Parents: 92bd50e Author: Aaron Tokhy Authored: Thu Aug 13 10:45:00 2015 -0800 Committer: Ashutosh Chauhan Committed: Mon Aug 24 10:15:20 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/HiveAlterHandler.java | 2 +- .../hadoop/hive/metastore/HiveMetaStore.java | 6 +----- .../hadoop/hive/metastore/MetaStoreUtils.java | 17 ++++++++++------- .../ql/metadata/SessionHiveMetaStoreClient.java | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f4aac7ef/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 5391171..f67f2df 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -234,7 +234,7 @@ public class HiveAlterHandler implements AlterHandler { Database db = msdb.getDatabase(newt.getDbName()); // Update table stats. For partitioned table, we update stats in // alterPartition() - MetaStoreUtils.updateUnpartitionedTableStatsFast(db, newt, wh, false, true); + MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true); } updateTableColumnStatsForAlterTable(msdb, oldt, newt); // now finally call alter table http://git-wip-us.apache.org/repos/asf/hive/blob/f4aac7ef/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 19a986d..aa0d99e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1406,11 +1406,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && !MetaStoreUtils.isView(tbl)) { - if (tbl.getPartitionKeysSize() == 0) { // Unpartitioned table - MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir); - } else { // Partitioned table with no partitions. - MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, true); - } + MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir); } // set create time http://git-wip-us.apache.org/repos/asf/hive/blob/f4aac7ef/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 38dc406..425549b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -169,19 +169,22 @@ public class MetaStoreUtils { return true; } - public static boolean updateUnpartitionedTableStatsFast(Database db, Table tbl, Warehouse wh, + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, boolean madeDir) throws MetaException { - return updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir, false); + return updateTableStatsFast(db, tbl, wh, madeDir, false); } - public static boolean updateUnpartitionedTableStatsFast(Database db, Table tbl, Warehouse wh, + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute) throws MetaException { - return updateUnpartitionedTableStatsFast(tbl, - wh.getFileStatusesForUnpartitionedTable(db, tbl), madeDir, forceRecompute); + FileStatus[] fileStatuses = {}; + if (tbl.getPartitionKeysSize() == 0) { // Update stats only when unpartitioned + fileStatuses = wh.getFileStatusesForUnpartitionedTable(db, tbl); + } + return updateTableStatsFast(tbl, fileStatuses, madeDir, forceRecompute); } /** - * Updates the numFiles and totalSize parameters for the passed unpartitioned Table by querying + * Updates the numFiles and totalSize parameters for the passed Table by querying * the warehouse if the passed Table does not already have values for these parameters. * @param tbl * @param fileStatus @@ -190,7 +193,7 @@ public class MetaStoreUtils { * these parameters set * @return true if the stats were updated, false otherwise */ - public static boolean updateUnpartitionedTableStatsFast(Table tbl, + public static boolean updateTableStatsFast(Table tbl, FileStatus[] fileStatus, boolean newDir, boolean forceRecompute) throws MetaException { Map params = tbl.getParameters(); http://git-wip-us.apache.org/repos/asf/hive/blob/f4aac7ef/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index d20f196..51ff262 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -386,7 +386,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I } org.apache.hadoop.hive.metastore.api.Table newtCopy = deepCopyAndLowerCaseTable(newt); - MetaStoreUtils.updateUnpartitionedTableStatsFast(newtCopy, + MetaStoreUtils.updateTableStatsFast(newtCopy, getWh().getFileStatusesForSD(newtCopy.getSd()), false, true); Table newTable = new Table(newtCopy); String newDbName = newTable.getDbName();