hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r768158 - in /hadoop/hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientnegative/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientnegative/ ...
Date Fri, 24 Apr 2009 06:14:03 GMT
Author: zshao
Date: Fri Apr 24 06:14:03 2009
New Revision: 768158

URL: http://svn.apache.org/viewvc?rev=768158&view=rev
Log:
HIVE-376. In strict mode do not allow join without "ON" condition. (Namit Jain via zshao)

Added:
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q
    hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hadoop/hive/trunk/conf/hive-default.xml
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Apr 24 06:14:03 2009
@@ -18,6 +18,9 @@
     HIVE-250. Shared memory java dbm for map-side joins.
     (Joydeep Sen Sarma via zshao)
 
+    HIVE-376. In strict mode do not allow join without "ON" condition.
+    (Namit Jain via zshao)
+
   IMPROVEMENTS
     HIVE-389. Option to build without ivy (jssarma)
 

Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Apr 24
06:14:03 2009
@@ -104,7 +104,7 @@
     // for hive script operator
     HIVETABLENAME("hive.table.name", ""),
     HIVEPARTITIONNAME("hive.partition.name", ""),
-    HIVEPARTITIONPRUNER("hive.partition.pruning", "nonstrict"),
+    HIVEMAPREDMODE("hive.mapred.mode", "nonstrict"),
     HIVEALIAS("hive.alias", ""),
     HIVEMAPSIDEAGGREGATE("hive.map.aggr", "true"),
     HIVEGROUPBYSKEW("hive.groupby.skewindata", "false"),

Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Fri Apr 24 06:14:03 2009
@@ -101,6 +101,12 @@
 </property>
 
 <property>
+  <name>hive.mapred.mode</name>
+  <value>nonstrict</value>
+  <description>The mode in which the hive operations are being performed. In strict
mode, some risky queries are not allowed to run</description>
+</property>
+
+<property>
   <name>hive.exec.script.maxerrsize</name>
   <value>100000</value>
   <description>Maximum number of bytes a script is allowed to emit to standard error
(per map-reduce task). This prevents runaway scripts from filling logs partitions to capacity
</description>

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Fri Apr 24
06:14:03 2009
@@ -72,6 +72,7 @@
   ORDERBY_SORTBY_CONFLICT("Cannot have both Order By and Sort By Clauses"),
   CLUSTERBY_ORDERBY_CONFLICT("Cannot have both Cluster By and Order By Clauses"),
   NO_LIMIT_WITH_ORDERBY("In strict mode, limit must be specified if ORDER BY is present"),
+  NO_CARTESIAN_PRODUCT("In strict mode, cartesian product is not allowed. If you really want
to perform the operation, set hive.mapred.mode=nonstrict"),
   UNION_NOTIN_SUBQ("Top level Union is not supported currently; use a subquery for the union"),
   INVALID_INPUT_FORMAT_TYPE("Input Format must implement InputFormat"),
   INVALID_OUTPUT_FORMAT_TYPE("Output Format must implement HiveOutputFormat, otherwise it
should be either IgnoreKeyTextOutputFormat or SequenceFileOutputFormat"),

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri
Apr 24 06:14:03 2009
@@ -625,7 +625,7 @@
           
            ASTNode whexp = (ASTNode)qbp.getWhrForClause(clause);
            if (pruner.getTable().isPartitioned() &&
-               conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict")
&&
+               conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict") &&
                (whexp == null || !pruner.hasPartitionPredicate((ASTNode)whexp.getChild(0))))
{
              throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE.getMsg(whexp !=
null ? whexp : qbp.getSelForClause(clause), 
                                                                                 " for Alias
" + alias + " Table " + pruner.getTable().getName()));
@@ -2421,7 +2421,7 @@
         assert numReducers == 1;
         // in strict mode, in the presence of order by, limit must be specified
         Integer limit = qb.getParseInfo().getDestLimit(dest);
-        if (conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict")
&& limit == null)
+        if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict") &&
limit == null)
           throw new SemanticException(ErrorMsg.NO_LIMIT_WITH_ORDERBY.getMsg(sortExprs));
       }
     }
@@ -2588,9 +2588,20 @@
       }
     }
 
+    int numReds = -1;
+
+    // Use only 1 reducer in case of cartesian product
+    if (reduceKeys.size() == 0) {
+      numReds = 1;
+
+      // Cartesian product is not supported in strict mode
+      if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict"))
+        throw new SemanticException(ErrorMsg.NO_CARTESIAN_PRODUCT.getMsg());
+    }
+
     ReduceSinkOperator rsOp = (ReduceSinkOperator)putOpInsertMap(
       OperatorFactory.getAndMakeChild(
-        PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(),
-1), 
+        PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(),
numReds), 
         new RowSchema(outputRS.getColumnInfos()),
         child), outputRS);
     rsOp.setColumnExprMap(colExprMap);

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q Fri Apr 24 06:14:03
2009
@@ -0,0 +1,3 @@
+set hive.mapred.mode=strict;
+
+SELECT *  FROM src src1 JOIN src src2;

Modified: hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q Fri Apr 24 06:14:03
2009
@@ -1,4 +1,4 @@
-set hive.partition.pruning=strict;
+set hive.mapred.mode=strict;
 
 EXPLAIN
 SELECT src.key, src.value from src order by src.key;

Modified: hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q Fri Apr 24 06:14:03
2009
@@ -1,4 +1,4 @@
-set hive.partition.pruning=strict;
+set hive.mapred.mode=strict;
 
 EXPLAIN
 SELECT count(1) FROM srcPART;

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q Fri Apr 24 06:14:03 2009
@@ -0,0 +1,4 @@
+EXPLAIN
+SELECT *  FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10;
+
+SELECT *  FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10;

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out Fri Apr 24 06:14:03
2009
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: In strict mode, cartesian product is not allowed. If
you really want to perform the operation, set hive.mapred.mode=nonstrict

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out Fri Apr 24 06:14:03
2009
@@ -0,0 +1,162 @@
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE
(and (< (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src2) key) 10)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src2 
+            Reduce Output Operator
+              sort order: 
+              tag: 1
+              value expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+        src1 
+            Reduce Output Operator
+              sort order: 
+              tag: 0
+              value expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE.0} {VALUE.1}
+            1 {VALUE.0} {VALUE.1}
+          Filter Operator
+            predicate:
+                expr: ((UDFToDouble(0) < UDFToDouble(10)) and (UDFToDouble(2) < UDFToDouble(10)))
+                type: boolean
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: string
+                    expr: 1
+                    type: string
+                    expr: 2
+                    type: string
+                    expr: 3
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+5	val_5	2	val_2
+5	val_5	5	val_5
+5	val_5	9	val_9
+5	val_5	4	val_4
+5	val_5	8	val_8
+5	val_5	0	val_0
+5	val_5	5	val_5
+5	val_5	5	val_5
+5	val_5	0	val_0
+5	val_5	0	val_0
+0	val_0	2	val_2
+0	val_0	5	val_5
+0	val_0	9	val_9
+0	val_0	4	val_4
+0	val_0	8	val_8
+0	val_0	0	val_0
+0	val_0	5	val_5
+0	val_0	5	val_5
+0	val_0	0	val_0
+0	val_0	0	val_0
+8	val_8	2	val_2
+8	val_8	5	val_5
+8	val_8	9	val_9
+8	val_8	4	val_4
+8	val_8	8	val_8
+8	val_8	0	val_0
+8	val_8	5	val_5
+8	val_8	5	val_5
+8	val_8	0	val_0
+8	val_8	0	val_0
+0	val_0	2	val_2
+0	val_0	5	val_5
+0	val_0	9	val_9
+0	val_0	4	val_4
+0	val_0	8	val_8
+0	val_0	0	val_0
+0	val_0	5	val_5
+0	val_0	5	val_5
+0	val_0	0	val_0
+0	val_0	0	val_0
+2	val_2	2	val_2
+2	val_2	5	val_5
+2	val_2	9	val_9
+2	val_2	4	val_4
+2	val_2	8	val_8
+2	val_2	0	val_0
+2	val_2	5	val_5
+2	val_2	5	val_5
+2	val_2	0	val_0
+2	val_2	0	val_0
+5	val_5	2	val_2
+5	val_5	5	val_5
+5	val_5	9	val_9
+5	val_5	4	val_4
+5	val_5	8	val_8
+5	val_5	0	val_0
+5	val_5	5	val_5
+5	val_5	5	val_5
+5	val_5	0	val_0
+5	val_5	0	val_0
+5	val_5	2	val_2
+5	val_5	5	val_5
+5	val_5	9	val_9
+5	val_5	4	val_4
+5	val_5	8	val_8
+5	val_5	0	val_0
+5	val_5	5	val_5
+5	val_5	5	val_5
+5	val_5	0	val_0
+5	val_5	0	val_0
+0	val_0	2	val_2
+0	val_0	5	val_5
+0	val_0	9	val_9
+0	val_0	4	val_4
+0	val_0	8	val_8
+0	val_0	0	val_0
+0	val_0	5	val_5
+0	val_0	5	val_5
+0	val_0	0	val_0
+0	val_0	0	val_0
+9	val_9	2	val_2
+9	val_9	5	val_5
+9	val_9	9	val_9
+9	val_9	4	val_4
+9	val_9	8	val_8
+9	val_9	0	val_0
+9	val_9	5	val_5
+9	val_9	5	val_5
+9	val_9	0	val_0
+9	val_9	0	val_0
+4	val_4	2	val_2
+4	val_4	5	val_5
+4	val_4	9	val_9
+4	val_4	4	val_4
+4	val_4	8	val_8
+4	val_4	0	val_0
+4	val_4	5	val_5
+4	val_4	5	val_5
+4	val_4	0	val_0
+4	val_4	0	val_0



Mime
View raw message