hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1235864 [1/2] - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/index/ java/org/apache/hadoop/hive/ql/index/compact/ java/org/apache/hadoop/hive/ql/optimizer/ test/org/apache/hadoop/hive/ql/ test/queries/clientpositive/ test/results/...
Date Wed, 25 Jan 2012 18:42:06 GMT
Author: namit
Date: Wed Jan 25 18:42:06 2012
New Revision: 1235864

URL: http://svn.apache.org/viewvc?rev=1235864&view=rev
Log:
HIVE-2727 add a testcase for partitioned view on union and base tables have
index (He Yongqiang via namit)


Added:
    hive/trunk/ql/src/test/queries/clientpositive/union_view.q
    hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java Wed
Jan 25 18:42:06 2012
@@ -67,14 +67,14 @@ public class IndexMetadataChangeTask ext
           return 1;
         }
 
-        Path url = new Path(part.getDataLocation().toString());
+        Path url = new Path(part.getPartitionPath().toString());
         FileSystem fs = url.getFileSystem(conf);
         FileStatus fstat = fs.getFileStatus(url);
 
         part.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
         db.alterPartition(tbl.getTableName(), part);
       } else {
-        Path url = new Path(tbl.getDataLocation().toString());
+        Path url = new Path(tbl.getPath().toString());
         FileSystem fs = url.getFileSystem(conf);
         FileStatus fstat = fs.getFileStatus(url);
         tbl.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
Wed Jan 25 18:42:06 2012
@@ -63,6 +63,8 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
 
 public class CompactIndexHandler extends TableBasedIndexHandler {
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java Wed Jan 25
18:42:06 2012
@@ -85,6 +85,10 @@ public final class IndexUtils {
       queryPartitions = queryPartitionList.getUnknownPartns();
     }
 
+    if(queryPartitions == null) {
+      return null;
+    }
+
     for (Partition part : queryPartitions) {
       List<Table> sourceIndexTables = getIndexTables(hive, part, indexes);
       if (!containsPartition(hive, part, indexes)) {

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1235864&r1=1235863&r2=1235864&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed Jan 25 18:42:06 2012
@@ -905,6 +905,7 @@ public class QTestUtil {
         ".*invalidscheme:.*",
         ".*lastUpdateTime.*",
         ".*lastAccessTime.*",
+        ".*lastModifiedTime.*",
         ".*[Oo]wner.*",
         ".*CreateTime.*",
         ".*LastAccessTime.*",

Added: hive/trunk/ql/src/test/queries/clientpositive/union_view.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union_view.q?rev=1235864&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union_view.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union_view.q Wed Jan 25 18:42:06 2012
@@ -0,0 +1,84 @@
+CREATE TABLE src_union_1 (key int, value string) PARTITIONED BY (ds string);
+CREATE INDEX src_union_1_key_idx ON TABLE src_union_1(key) AS 'COMPACT' WITH DEFERRED REBUILD;
+
+CREATE TABLE src_union_2 (key int, value string) PARTITIONED BY (ds string, part_1 string);
+CREATE INDEX src_union_2_key_idx ON TABLE src_union_2(key) AS 'COMPACT' WITH DEFERRED REBUILD;
+
+CREATE TABLE src_union_3(key int, value string) PARTITIONED BY (ds string, part_1 string,
part_2 string);
+CREATE INDEX src_union_3_key_idx ON TABLE src_union_3(key) AS 'COMPACT' WITH DEFERRED REBUILD;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+SET hive.optimize.index.filter=true;
+SET hive.optimize.index.filter.compact.minsize=0;
+
+SET hive.exec.pre.hooks=;
+SET hive.exec.post.hooks=;
+SET hive.semantic.analyzer.hook=;
+SET hive.merge.mapfiles=false;
+SET hive.merge.mapredfiles=false;
+
+INSERT OVERWRITE TABLE src_union_1 PARTITION (ds='1') SELECT * FROM src;
+ALTER INDEX src_union_1_key_idx ON src_union_1 PARTITION (ds='1') REBUILD;
+
+INSERT OVERWRITE TABLE src_union_2 PARTITION (ds='2', part_1='1') SELECT * FROM src;
+INSERT OVERWRITE TABLE src_union_2 PARTITION (ds='2', part_1='2') SELECT * FROM src;
+ALTER INDEX src_union_2_key_idx ON src_union_2 PARTITION (ds='2', part_1='1') REBUILD;
+ALTER INDEX src_union_2_key_idx ON src_union_2 PARTITION (ds='2', part_1='2') REBUILD;
+
+INSERT OVERWRITE TABLE src_union_3 PARTITION (ds='3', part_1='1', part_2='2:3+4') SELECT
* FROM src;
+INSERT OVERWRITE TABLE src_union_3 PARTITION (ds='3', part_1='2', part_2='2:3+4') SELECT
* FROM src;
+ALTER INDEX src_union_3_key_idx ON src_union_3 PARTITION (ds='3', part_1='1', part_2='2:3+4')
REBUILD;
+ALTER INDEX src_union_3_key_idx ON src_union_3 PARTITION (ds='3', part_1='2', part_2='2:3+4')
REBUILD;
+
+EXPLAIN SELECT key, value, ds FROM src_union_1 WHERE key=86 and ds='1';
+EXPLAIN SELECT key, value, ds FROM src_union_2 WHERE key=86 and ds='2';
+EXPLAIN SELECT key, value, ds FROM src_union_3 WHERE key=86 and ds='3';
+
+SELECT key, value, ds FROM src_union_1 WHERE key=86 AND ds ='1';
+SELECT key, value, ds FROM src_union_2 WHERE key=86 AND ds ='2';
+SELECT key, value, ds FROM src_union_3 WHERE key=86 AND ds ='3';
+
+EXPLAIN SELECT count(1) from src_union_1 WHERE ds ='1';
+EXPLAIN SELECT count(1) from src_union_2 WHERE ds ='2';
+EXPLAIN SELECT count(1) from src_union_3 WHERE ds ='3';
+
+SELECT count(1) from src_union_1 WHERE ds ='1';
+SELECT count(1) from src_union_2 WHERE ds ='2';
+SELECT count(1) from src_union_3 WHERE ds ='3';
+
+CREATE VIEW src_union_view PARTITIONED ON (ds) as
+SELECT key, value, ds FROM (
+SELECT key, value, ds FROM src_union_1
+UNION ALL
+SELECT key, value, ds FROM src_union_2
+UNION ALL
+SELECT key, value, ds FROM src_union_3
+) subq;
+
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='1';
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='2';
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='3';
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds IS NOT NULL order by
ds;
+
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='1';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='2';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='3';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds IS NOT NULL order by ds;
+
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='1';
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='2';
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='3';
+
+SELECT count(1) from src_union_view WHERE ds ='1';
+SELECT count(1) from src_union_view WHERE ds ='2';
+SELECT count(1) from src_union_view WHERE ds ='3';
+
+INSERT OVERWRITE TABLE src_union_3 PARTITION (ds='4', part_1='1', part_2='2:3+4') SELECT
* FROM src;
+ALTER INDEX src_union_3_key_idx ON src_union_3 PARTITION (ds='4', part_1='1', part_2='2:3+4')
REBUILD;
+
+EXPLAIN SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='4';
+SELECT key, value, ds FROM src_union_view WHERE key=86 AND ds ='4';
+
+EXPLAIN SELECT count(1) from src_union_view WHERE ds ='4';
+SELECT count(1) from src_union_view WHERE ds ='4';



Mime
View raw message