impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sail...@apache.org
Subject [5/6] incubator-impala git commit: IMPALA-4502: test_partition_ddl_predicates breaks on non-HDFS filesystems
Date Tue, 22 Nov 2016 03:05:57 GMT
IMPALA-4502: test_partition_ddl_predicates breaks on non-HDFS filesystems

This is because that test uses 'set cached' and 'set uncached' which
are not supported on non-HDFS filesystems. This patch creates a
separate test file for non-HDFS filesystems with only supported
queries and invokes the right file based on the filesystem.

Change-Id: I8606aa427cb6e50be3395cdde246abb53db5172c
Reviewed-on: http://gerrit.cloudera.org:8080/5164
Reviewed-by: Sailesh Mukil <sailesh@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/178fd591
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/178fd591
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/178fd591

Branch: refs/heads/master
Commit: 178fd59142db5f401d1426fa4daa11432e1e2f50
Parents: 7bfa6e3
Author: Sailesh Mukil <sailesh@cloudera.com>
Authored: Mon Nov 21 08:27:21 2016 -0800
Committer: Internal Jenkins <cloudera-hudson@gerrit.cloudera.org>
Committed: Tue Nov 22 00:42:57 2016 +0000

----------------------------------------------------------------------
 .../partition-ddl-predicates-all-fs.test        | 135 ++++++++++++++++
 .../partition-ddl-predicates-hdfs-only.test     |  38 +++++
 .../QueryTest/partition-ddl-predicates.test     | 156 -------------------
 tests/metadata/test_ddl.py                      |   7 +-
 4 files changed, 178 insertions(+), 158 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
new file mode 100644
index 0000000..36104c3
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
@@ -0,0 +1,135 @@
+====
+---- QUERY
+# First create a partitioned table
+create table p1 (i int) partitioned by(j int, k string);
+# Add some partitions
+alter table p1 add partition (j=1,k="a");
+alter table p1 add partition (j=1,k="b");
+alter table p1 add partition (j=1,k="c");
+alter table p1 add partition (j=2,k="d");
+alter table p1 add partition (j=2,k="e");
+alter table p1 add partition (j=2,k="f");
+alter table p1 add partition (j=2,k=NULL);
+alter table p1 add partition (j=NULL,k="g");
+alter table p1 add partition (j=NULL,k=NULL);
+# Populate the table
+insert into p1 partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c");
+====
+---- QUERY
+show partitions p1
+---- RESULTS
+'NULL','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__
+'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
+'1','a',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=a
+'1','b',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=b
+'1','c',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=c
+'2','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=__HIVE_DEFAULT_PARTITION__
+'2','d',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
+'2','e',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
+'2','f',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
+'Total','',-1,3,regex:.+,regex:.+,'','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show files in p1 partition (j<2, k="a")
+---- RESULTS
+regex:.+,regex:.+,'j=1/k=a'
+---- TYPES
+STRING, STRING, STRING
+====
+---- QUERY
+alter table p1 partition (j=2) set fileformat parquet
+---- RESULTS
+'Updated 4 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (k rlike ".*") set serdeproperties ('k'='v')
+---- RESULTS
+'Updated 7 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (k="") set serdeproperties ('k'='v')
+---- RESULTS
+'Updated 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (k=null) set serdeproperties ('k'='v')
+---- RESULTS
+'Updated 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (j is not null) set tblproperties ('k'='v')
+---- RESULTS
+'Updated 7 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+compute incremental stats p1 partition(j<2)
+---- RESULTS
+'Updated 3 partition(s) and 1 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j<2)
+---- RESULTS
+'Dropped 3 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j<2)
+---- RESULTS
+'Dropped 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop if exists partition (j<2)
+---- RESULTS
+'Dropped 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop if exists partition (j=2, k="bla")
+---- RESULTS
+'Dropped 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j=2, k=NULL)
+---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j IS NULL AND k IS NULL);
+---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show partitions p1
+---- RESULTS
+'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
+'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
+'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
+'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
+'Total','',0,0,regex:.+,regex:.+,'','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test
b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test
new file mode 100644
index 0000000..5ca2a06
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-hdfs-only.test
@@ -0,0 +1,38 @@
+====
+---- QUERY
+# First create a partitioned table
+create table p1_hdfs (i int) partitioned by(j int, k string);
+# Add some partitions
+alter table p1_hdfs add partition (j=1,k="a");
+alter table p1_hdfs add partition (j=1,k="b");
+alter table p1_hdfs add partition (j=1,k="c");
+alter table p1_hdfs add partition (j=2,k="d");
+alter table p1_hdfs add partition (j=2,k="e");
+alter table p1_hdfs add partition (j=2,k="f");
+alter table p1_hdfs add partition (j=2,k=NULL);
+alter table p1_hdfs add partition (j=NULL,k="g");
+alter table p1_hdfs add partition (j=NULL,k=NULL);
+# Populate the table
+insert into p1_hdfs partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c");
+====
+---- QUERY
+alter table p1_hdfs partition (j<2, k in ("b", "c")) set cached in 'testPool'
+---- RESULTS
+'Cached 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1_hdfs partition (j<2, j>0, k<>"d") set uncached
+---- RESULTS
+'Uncached 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1_hdfs partition (j=3 or j=2, k like "%") set uncached
+---- RESULTS
+'Uncached 0 partition(s).'
+---- TYPES
+STRING
+====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
deleted file mode 100644
index cc7e865..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
+++ /dev/null
@@ -1,156 +0,0 @@
-====
----- QUERY
-# First create a partitioned table
-create table p1 (i int) partitioned by(j int, k string);
-# Add some partitions
-alter table p1 add partition (j=1,k="a");
-alter table p1 add partition (j=1,k="b");
-alter table p1 add partition (j=1,k="c");
-alter table p1 add partition (j=2,k="d");
-alter table p1 add partition (j=2,k="e");
-alter table p1 add partition (j=2,k="f");
-alter table p1 add partition (j=2,k=NULL);
-alter table p1 add partition (j=NULL,k="g");
-alter table p1 add partition (j=NULL,k=NULL);
-# Populate the table
-insert into p1 partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c");
-====
----- QUERY
-show partitions p1
----- RESULTS
-'NULL','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__
-'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
-'1','a',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=a
-'1','b',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=b
-'1','c',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=c
-'2','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=__HIVE_DEFAULT_PARTITION__
-'2','d',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
-'2','e',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
-'2','f',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
-'Total','',-1,3,regex:.+,regex:.+,'','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show files in p1 partition (j<2, k="a")
----- RESULTS
-regex:.+,regex:.+,'j=1/k=a'
----- TYPES
-STRING, STRING, STRING
-====
----- QUERY
-alter table p1 partition (j<2, k in ("b", "c")) set cached in 'testPool'
----- RESULTS
-'Cached 2 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (j<2, j>0, k<>"d") set uncached
----- RESULTS
-'Uncached 2 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (j=3 or j=2, k like "%") set uncached
----- RESULTS
-'Uncached 0 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (j=2) set fileformat parquet
----- RESULTS
-'Updated 4 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (k rlike ".*") set serdeproperties ('k'='v')
----- RESULTS
-'Updated 7 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (k="") set serdeproperties ('k'='v')
----- RESULTS
-'Updated 2 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (k=null) set serdeproperties ('k'='v')
----- RESULTS
-'Updated 2 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 partition (j is not null) set tblproperties ('k'='v')
----- RESULTS
-'Updated 7 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-compute incremental stats p1 partition(j<2)
----- RESULTS
-'Updated 3 partition(s) and 1 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 drop partition (j<2)
----- RESULTS
-'Dropped 3 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 drop partition (j<2)
----- RESULTS
-'Dropped 0 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 drop if exists partition (j<2)
----- RESULTS
-'Dropped 0 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 drop if exists partition (j=2, k="bla")
----- RESULTS
-'Dropped 0 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 drop partition (j=2, k=NULL)
----- RESULTS
-'Dropped 1 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-alter table p1 drop partition (j IS NULL AND k IS NULL);
----- RESULTS
-'Dropped 1 partition(s).'
----- TYPES
-STRING
-====
----- QUERY
-show partitions p1
----- RESULTS
-'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
-'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
-'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
-'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
-'Total','',0,0,regex:.+,regex:.+,'','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/178fd591/tests/metadata/test_ddl.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 15fe4c7..58f42eb 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -24,7 +24,7 @@ from tests.common.impala_test_suite import LOG
 from tests.common.parametrize import UniqueDatabase
 from tests.common.skip import SkipIf, SkipIfLocal, SkipIfOldAggsJoins
 from tests.common.test_dimensions import create_single_exec_option_dimension
-from tests.util.filesystem_utils import WAREHOUSE, IS_LOCAL, IS_S3
+from tests.util.filesystem_utils import WAREHOUSE, IS_HDFS, IS_LOCAL, IS_S3
 
 # Validates DDL statements (create, drop)
 class TestDdlStatements(TestDdlBase):
@@ -393,8 +393,11 @@ class TestDdlStatements(TestDdlBase):
 
   @UniqueDatabase.parametrize(sync_ddl=True)
   def test_partition_ddl_predicates(self, vector, unique_database):
-    self.run_test_case('QueryTest/partition-ddl-predicates', vector,
+    self.run_test_case('QueryTest/partition-ddl-predicates-all-fs', vector,
         use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
+    if IS_HDFS:
+      self.run_test_case('QueryTest/partition-ddl-predicates-hdfs-only', vector,
+          use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
 
 # IMPALA-2002: Tests repeated adding/dropping of .jar and .so in the lib cache.
 class TestLibCache(TestDdlBase):


Mime
View raw message