impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [08/33] incubator-impala git commit: IMPALA-4155: Update default partition when table is altered
Date Tue, 25 Oct 2016 20:15:55 GMT
IMPALA-4155: Update default partition when table is altered

If the table format is changed by the Alter Table statement, the
default partition in partitioned tables used to not get updated. This
caused a problem because Insert picks up the file format for new
partitions from the default partition. This patch fixes the problem by
calling addDefaultPartition().

Also removed "drop table if not exists" in tests in alter-table.test
because we already have the unique_database fixture.

Change-Id: I59bf21caa5c5e7867d07d87cda0c0a5b4b994859
Reviewed-on: http://gerrit.cloudera.org:8080/4750
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bf1d9677
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bf1d9677
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bf1d9677

Branch: refs/heads/hadoop-next
Commit: bf1d9677fc09dabc88ab6c71f82ec99ec59ec164
Parents: c6fc899
Author: Taras Bobrovytsky <tbobrovytsky@cloudera.com>
Authored: Mon Oct 10 12:14:52 2016 -0700
Committer: Internal Jenkins <cloudera-hudson@gerrit.cloudera.org>
Committed: Thu Oct 20 23:47:52 2016 +0000

----------------------------------------------------------------------
 .../org/apache/impala/catalog/HdfsTable.java    | 29 ++++++++-------
 .../impala/service/CatalogOpExecutor.java       |  3 ++
 .../queries/QueryTest/alter-table.test          | 38 ++++++++++----------
 3 files changed, 39 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf1d9677/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index e664f6f..e6206fc 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -965,7 +965,10 @@ public class HdfsTable extends Table {
     return partition;
   }
 
-  private void addDefaultPartition(StorageDescriptor storageDescriptor)
+  /**
+   * Adds or replaces the default partition.
+   */
+  public void addDefaultPartition(StorageDescriptor storageDescriptor)
       throws CatalogException {
     // Default partition has no files and is not referred to by scan nodes. Data sinks
     // refer to this to understand how to create new partitions.
@@ -1025,15 +1028,7 @@ public class HdfsTable extends Table {
         return;
       }
       // Load partition and file metadata
-      if (!reuseMetadata) {
-        // Load all partitions from Hive Metastore, including file metadata.
-        LOG.debug("load table from Hive Metastore: " + db_.getName() + "." + name_);
-        List<org.apache.hadoop.hive.metastore.api.Partition> msPartitions =
-            Lists.newArrayList();
-        msPartitions.addAll(MetaStoreUtil.fetchAllPartitions(
-            client, db_.getName(), name_, NUM_PARTITION_FETCH_RETRIES));
-        loadAllPartitions(msPartitions, msTbl);
-      } else {
+      if (reuseMetadata) {
         // Incrementally update this table's partitions and file metadata
         LOG.debug("incremental update for table: " + db_.getName() + "." + name_);
         Preconditions.checkState(partitionsToUpdate == null || loadFileMetadata);
@@ -1043,6 +1038,14 @@ public class HdfsTable extends Table {
         } else {
           updatePartitionsFromHms(client, partitionsToUpdate, loadFileMetadata);
         }
+      } else {
+        // Load all partitions from Hive Metastore, including file metadata.
+        LOG.debug("load table from Hive Metastore: " + db_.getName() + "." + name_);
+        List<org.apache.hadoop.hive.metastore.api.Partition> msPartitions =
+            Lists.newArrayList();
+        msPartitions.addAll(MetaStoreUtil.fetchAllPartitions(
+            client, db_.getName(), name_, NUM_PARTITION_FETCH_RETRIES));
+        loadAllPartitions(msPartitions, msTbl);
       }
       if (loadTableSchema) setAvroSchema(client, msTbl);
       updateStatsFromHmsTable(msTbl);
@@ -1420,9 +1423,9 @@ public class HdfsTable extends Table {
     HdfsStorageDescriptor fileFormatDescriptor =
         HdfsStorageDescriptor.fromStorageDescriptor(this.name_, msTbl.getSd());
     Map<FsKey, FileBlocksInfo> perFsFileBlocks = Maps.newHashMap();
-    for (HdfsPartition part: partitions) {
+    for (HdfsPartition partition: partitions) {
       org.apache.hadoop.hive.metastore.api.Partition msPart =
-          part.toHmsPartition();
+          partition.toHmsPartition();
       StorageDescriptor sd = null;
       if (msPart == null) {
         // If this partition is not stored in the Hive Metastore (e.g. default partition
@@ -1432,7 +1435,7 @@ public class HdfsTable extends Table {
       } else {
         sd = msPart.getSd();
       }
-      loadPartitionFileMetadata(sd, part, fileFormatDescriptor.getFileFormat(),
+      loadPartitionFileMetadata(sd, partition, fileFormatDescriptor.getFileFormat(),
           perFsFileBlocks);
     }
     loadDiskIds(perFsFileBlocks);

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf1d9677/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 780bf13..5743a59 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -1869,6 +1869,9 @@ public class CatalogOpExecutor {
       org.apache.hadoop.hive.metastore.api.Table msTbl =
           tbl.getMetaStoreTable().deepCopy();
       setStorageDescriptorFileFormat(msTbl.getSd(), fileFormat);
+      // The default partition must be updated if the file format is changed so that new
+      // partitions are created with the new file format.
+      if (tbl instanceof HdfsTable) ((HdfsTable) tbl).addDefaultPartition(msTbl.getSd());
       applyAlterTable(msTbl);
       reloadFileMetadata = true;
     } else {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf1d9677/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
index 1f63c2b..3b1d86b 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
@@ -612,9 +612,6 @@ STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING,
STRING
 ====
 ---- QUERY
 # IMPALA-1016: Testing scanning newly added columns
-DROP TABLE IF EXISTS imp1016
-====
----- QUERY
 CREATE TABLE imp1016 (string1 string)
 ---- RESULTS
 ====
@@ -672,9 +669,6 @@ bigint
 ====
 ---- QUERY
 # Create a larger table to test scanning newly added columns
-DROP TABLE IF EXISTS imp1016Large
-====
----- QUERY
 CREATE TABLE imp1016Large (string1 string)
 ---- RESULTS
 ====
@@ -929,30 +923,38 @@ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
STRING,
 ====
 ---- QUERY
 # IMPALA-1740: Test setting the skip.header.line.count tblproperty
-drop table if exists i1740_alter;
-create table i1740_alter (i1 integer);
-alter table i1740_alter set tblproperties ('skip.header.line.count'='2');
+create table i1740_alter_1 (i1 integer);
+alter table i1740_alter_1 set tblproperties ('skip.header.line.count'='2');
 ---- RESULTS
 ====
 ---- QUERY
 # IMPALA-1740: Test setting the skip.header.line.count tblproperty
-drop table if exists i1740_alter;
-create table i1740_alter (i1 integer);
-alter table i1740_alter set tblproperties ('skip.header.line.count'='-1');
+create table i1740_alter_2 (i1 integer);
+alter table i1740_alter_2 set tblproperties ('skip.header.line.count'='-1');
 ---- CATCH
 Invalid value for table property skip.header.line.count:
 ====
 ---- QUERY
 # IMPALA-1740: Test setting the skip.header.line.count tblproperty
-drop table if exists i1740_alter;
-create table i1740_alter (i1 integer);
-alter table i1740_alter set tblproperties ('skip.header.line.count'='foo');
+create table i1740_alter_3 (i1 integer);
+alter table i1740_alter_3 set tblproperties ('skip.header.line.count'='foo');
 ---- CATCH
 Invalid value for table property skip.header.line.count:
 ====
 ---- QUERY
 # IMPALA-1740: Test setting the skip.header.line.count tblproperty
-drop table if exists i1740_alter;
-create table i1740_alter (i1 integer) stored as parquet;
-alter table i1740_alter set tblproperties ('skip.header.line.count'='2');
+create table i1740_alter_4 (i1 integer) stored as parquet;
+alter table i1740_alter_4 set tblproperties ('skip.header.line.count'='2');
+====
+---- QUERY
+# IMPALA-4155: Verify that the default partition is updated by the alter table statement
+create table i4155_alter (c1 int)
+partitioned by (p1 string) row format delimited fields terminated by ',';
+alter table i4155_alter set fileformat PARQUET;
+insert into i4155_alter partition(p1="abc") values (1);
+select * from i4155_alter;
+---- RESULTS
+1,'abc'
+---- TYPES
+INT, STRING
 ====


Mime
View raw message