Return-Path: 
 <commits-return-9832-apmail-hive-commits-archive=hive.apache.org@hive.apache.org>
X-Original-To: apmail-hive-commits-archive@www.apache.org
Delivered-To: apmail-hive-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id 1DF7811829
	for <apmail-hive-commits-archive@www.apache.org>;
 Wed, 24 Sep 2014 07:12:24 +0000 (UTC)
Received: (qmail 71252 invoked by uid 500); 24 Sep 2014 07:12:23 -0000
Delivered-To: apmail-hive-commits-archive@hive.apache.org
Received: (qmail 71209 invoked by uid 500); 24 Sep 2014 07:12:23 -0000
Mailing-List: contact commits-help@hive.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@hive.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@hive.apache.org>
List-Post: <mailto:commits@hive.apache.org>
List-Id: <commits.hive.apache.org>
Reply-To: hive-dev@hive.apache.org
Delivered-To: mailing list commits@hive.apache.org
Received: (qmail 71198 invoked by uid 99); 24 Sep 2014 07:12:23 -0000
Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Sep 2014 07:12:23 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=5.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Sep 2014 07:11:59 +0000
Received: from eris.apache.org (localhost [127.0.0.1])
	by eris.apache.org (Postfix) with ESMTP id BC7F323888A6;
	Wed, 24 Sep 2014 07:11:57 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1627237 - in /hive/trunk: itests/src/test/resources/
 ql/src/java/org/apache/hadoop/hive/ql/exec/
 ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez/
Date: Wed, 24 Sep 2014 07:11:57 -0000
To: commits@hive.apache.org
From: gunther@apache.org
X-Mailer: svnmailer-1.0.9
Message-Id: <20140924071157.BC7F323888A6@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Author: gunther
Date: Wed Sep 24 07:11:57 2014
New Revision: 1627237

URL: http://svn.apache.org/r1627237
Log:
HIVE-8227: NPE w/ hive on tez when doing unions on empty tables (Gunther Hagleitner, reviewed by Vikram Dixit K)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/tez_union_group_by.q
    hive/trunk/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out
Modified:
    hive/trunk/itests/src/test/resources/testconfiguration.properties
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java

Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1627237&r1=1627236&r2=1627237&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Wed Sep 24 07:11:57 2014
@@ -204,6 +204,7 @@ minitez.query.files=bucket_map_join_tez1
   tez_schema_evolution.q,\
   tez_union.q,\
   tez_union_decimal.q,\
+  tez_union_group_by.q,\
   tez_smb_main.q,\
   tez_smb_1.q,\
   vectorized_dynamic_partition_pruning.q

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java?rev=1627237&r1=1627236&r2=1627237&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java Wed Sep 24 07:11:57 2014
@@ -46,6 +46,9 @@ public class OperatorUtils {
   public static <T> Set<T> findOperators(Collection<Operator<?>> starts, Class<T> clazz) {
     Set<T> found = new HashSet<T>();
     for (Operator<?> start : starts) {
+      if (start == null) {
+        continue;
+      }
       findOperators(start, clazz, found);
     }
     return found;

Added: hive/trunk/ql/src/test/queries/clientpositive/tez_union_group_by.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/tez_union_group_by.q?rev=1627237&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/tez_union_group_by.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/tez_union_group_by.q Wed Sep 24 07:11:57 2014
@@ -0,0 +1,87 @@
+CREATE TABLE x
+(
+u bigint,
+t string,
+st string
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB");
+
+CREATE TABLE y
+(
+u bigint
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB");
+
+CREATE TABLE z
+(
+u bigint
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB");
+
+CREATE TABLE v
+(
+t string, 
+st string,
+id int
+)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB");
+
+EXPLAIN 
+SELECT o.u, n.u
+FROM 
+(
+SELECT m.u, Min(date) as ft
+FROM 
+(
+SELECT u, date FROM x WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM y WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM z WHERE date < '2014-09-02' 
+) m
+GROUP BY m.u
+) n 
+LEFT OUTER JOIN
+(
+SELECT x.u
+FROM x
+JOIN v 
+ON (x.t = v.t AND x.st <=> v.st)
+WHERE x.date >= '2014-03-04' AND x.date < '2014-09-03'
+GROUP BY x.u
+) o
+ON n.u = o.u 
+WHERE n.u <> 0 AND n.ft <= '2014-09-02';
+
+SELECT o.u, n.u
+FROM 
+(
+SELECT m.u, Min(date) as ft
+FROM 
+(
+SELECT u, date FROM x WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM y WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM z WHERE date < '2014-09-02' 
+) m
+GROUP BY m.u
+) n 
+LEFT OUTER JOIN
+(
+SELECT x.u
+FROM x
+JOIN v 
+ON (x.t = v.t AND x.st <=> v.st)
+WHERE x.date >= '2014-03-04' AND x.date < '2014-09-03'
+GROUP BY x.u
+) o
+ON n.u = o.u 
+WHERE n.u <> 0 AND n.ft <= '2014-09-02';

Added: hive/trunk/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out?rev=1627237&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/tez_union_group_by.q.out Wed Sep 24 07:11:57 2014
@@ -0,0 +1,327 @@
+PREHOOK: query: CREATE TABLE x
+(
+u bigint,
+t string,
+st string
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@x
+POSTHOOK: query: CREATE TABLE x
+(
+u bigint,
+t string,
+st string
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@x
+PREHOOK: query: CREATE TABLE y
+(
+u bigint
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@y
+POSTHOOK: query: CREATE TABLE y
+(
+u bigint
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@y
+PREHOOK: query: CREATE TABLE z
+(
+u bigint
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@z
+POSTHOOK: query: CREATE TABLE z
+(
+u bigint
+)
+PARTITIONED BY (date string)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@z
+PREHOOK: query: CREATE TABLE v
+(
+t string, 
+st string,
+id int
+)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: CREATE TABLE v
+(
+t string, 
+st string,
+id int
+)
+STORED AS ORC 
+TBLPROPERTIES ("orc.compress"="ZLIB")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+PREHOOK: query: EXPLAIN 
+SELECT o.u, n.u
+FROM 
+(
+SELECT m.u, Min(date) as ft
+FROM 
+(
+SELECT u, date FROM x WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM y WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM z WHERE date < '2014-09-02' 
+) m
+GROUP BY m.u
+) n 
+LEFT OUTER JOIN
+(
+SELECT x.u
+FROM x
+JOIN v 
+ON (x.t = v.t AND x.st <=> v.st)
+WHERE x.date >= '2014-03-04' AND x.date < '2014-09-03'
+GROUP BY x.u
+) o
+ON n.u = o.u 
+WHERE n.u <> 0 AND n.ft <= '2014-09-02'
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN 
+SELECT o.u, n.u
+FROM 
+(
+SELECT m.u, Min(date) as ft
+FROM 
+(
+SELECT u, date FROM x WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM y WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM z WHERE date < '2014-09-02' 
+) m
+GROUP BY m.u
+) n 
+LEFT OUTER JOIN
+(
+SELECT x.u
+FROM x
+JOIN v 
+ON (x.t = v.t AND x.st <=> v.st)
+WHERE x.date >= '2014-03-04' AND x.date < '2014-09-03'
+GROUP BY x.u
+) o
+ON n.u = o.u 
+WHERE n.u <> 0 AND n.ft <= '2014-09-02'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+      Edges:
+        Map 10 <- Union 7 (CONTAINS)
+        Map 6 <- Union 7 (CONTAINS)
+        Map 9 <- Union 7 (CONTAINS)
+        Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+        Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+        Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+        Reducer 8 <- Union 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: v
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Filter Operator
+                    predicate: t is not null (type: boolean)
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: t (type: string), st (type: string)
+                      sort order: ++
+                      Map-reduce partition columns: t (type: string), st (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+        Map 10 
+        Map 5 
+        Map 6 
+        Map 9 
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0}
+                  1 
+                nullSafes: [false, true]
+                outputColumnNames: _col0
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: bigint)
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Group By Operator
+                    keys: _col0 (type: bigint)
+                    mode: hash
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: bigint)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: bigint)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+        Reducer 3 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: bigint)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: bigint)
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: bigint)
+                    sort order: +
+                    Map-reduce partition columns: _col0 (type: bigint)
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+        Reducer 4 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Left Outer Join0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0}
+                  1 {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col2
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col2 (type: bigint), _col0 (type: bigint)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 8 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: min(VALUE._col0)
+                keys: KEY._col0 (type: bigint)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Filter Operator
+                  predicate: (_col1 <= '2014-09-02') (type: boolean)
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: bigint)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: bigint)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: bigint)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+        Union 7 
+            Vertex: Union 7
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: SELECT o.u, n.u
+FROM 
+(
+SELECT m.u, Min(date) as ft
+FROM 
+(
+SELECT u, date FROM x WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM y WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM z WHERE date < '2014-09-02' 
+) m
+GROUP BY m.u
+) n 
+LEFT OUTER JOIN
+(
+SELECT x.u
+FROM x
+JOIN v 
+ON (x.t = v.t AND x.st <=> v.st)
+WHERE x.date >= '2014-03-04' AND x.date < '2014-09-03'
+GROUP BY x.u
+) o
+ON n.u = o.u 
+WHERE n.u <> 0 AND n.ft <= '2014-09-02'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@v
+PREHOOK: Input: default@x
+PREHOOK: Input: default@y
+PREHOOK: Input: default@z
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT o.u, n.u
+FROM 
+(
+SELECT m.u, Min(date) as ft
+FROM 
+(
+SELECT u, date FROM x WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM y WHERE date < '2014-09-02' 
+UNION ALL
+SELECT u, date FROM z WHERE date < '2014-09-02' 
+) m
+GROUP BY m.u
+) n 
+LEFT OUTER JOIN
+(
+SELECT x.u
+FROM x
+JOIN v 
+ON (x.t = v.t AND x.st <=> v.st)
+WHERE x.date >= '2014-03-04' AND x.date < '2014-09-03'
+GROUP BY x.u
+) o
+ON n.u = o.u 
+WHERE n.u <> 0 AND n.ft <= '2014-09-02'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@v
+POSTHOOK: Input: default@x
+POSTHOOK: Input: default@y
+POSTHOOK: Input: default@z
+#### A masked pattern was here ####