hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r766409 - in /hadoop/hive/branches/branch-0.3: CHANGES.txt ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/test/queries/clientpositive/input4_limit.q ql/src/test/results/clientpositive/input4_limit.q.out
Date Sun, 19 Apr 2009 00:54:38 GMT
Author: zshao
Date: Sun Apr 19 00:54:37 2009
New Revision: 766409

URL: http://svn.apache.org/viewvc?rev=766409&view=rev
Log:
HIVE-404. Fix ordering in "SELECT * FROM t SORT BY col1 LIMIT 100" when query is a outer-most
query.  (Namit Jain via zshao)

Added:
    hadoop/hive/branches/branch-0.3/ql/src/test/queries/clientpositive/input4_limit.q
    hadoop/hive/branches/branch-0.3/ql/src/test/results/clientpositive/input4_limit.q.out
Modified:
    hadoop/hive/branches/branch-0.3/CHANGES.txt
    hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

Modified: hadoop/hive/branches/branch-0.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/CHANGES.txt?rev=766409&r1=766408&r2=766409&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.3/CHANGES.txt Sun Apr 19 00:54:37 2009
@@ -25,10 +25,10 @@
     HIVE-393. Remove unnecessary checks for file type.
     (Zheng Shao via namit)
 
-    HIVE-82 Support for building tarballs and javadocs
+    HIVE-82. Support for building tarballs and javadocs
     (Ashish Thusoo via rmurthy)
 
-    HIVE-407 Permission issues in test scripts in tarballs.
+    HIVE-407. Permission issues in test scripts in tarballs.
     (Ashish Thusoo via namit)
 
   IMPROVEMENTS
@@ -152,6 +152,9 @@
 
     HIVE-421. Fix union followed by multi-table insert. (Namit Jain via zshao).
 
+    HIVE-404. Fix ordering in "SELECT * FROM t SORT BY col1 LIMIT 100" when
+    query is a outer-most query.  (Namit Jain via zshao)
+
 Release 0.2.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=766409&r1=766408&r2=766409&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++ hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Sun Apr 19 00:54:37 2009
@@ -2309,8 +2309,8 @@
     // Add the limit operator to get the value fields
     Operator curr = genLimitPlan(dest, qb, input, limit);
 
-    // If it is a outer most query, the exact limit is applied by the fetch task
-    if (isOuterQuery)
+    // If it is a outer most query and no sorting is specified, exact limit is applied by
the fetch task
+    if (isOuterQuery && !sortRequired(dest, qb))
       return curr;
 
     // Create a reduceSink operator followed by another limit
@@ -2318,6 +2318,21 @@
     return genLimitPlan(dest, qb, curr, limit);
   }
 
+  /*
+   * Is sorting reuired ?
+   * If there are no cluster by/sort by keys, then an additional map-reduce job is not needed.
+   * Else, sort the output by the relevant key (via another map-reduce job).
+   */
+  private boolean sortRequired(String dest, QB qb) {
+    if (qb.getParseInfo().getClusterByForClause(dest) != null)
+      return true;
+
+    if (qb.getParseInfo().getSortByForClause(dest) != null)
+      return true;
+    
+    return false;
+  }
+
   @SuppressWarnings("nls")
   private Operator genReduceSinkPlan(String dest, QB qb,
                                      Operator input, int numReducers) throws SemanticException
{

Added: hadoop/hive/branches/branch-0.3/ql/src/test/queries/clientpositive/input4_limit.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/test/queries/clientpositive/input4_limit.q?rev=766409&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/test/queries/clientpositive/input4_limit.q (added)
+++ hadoop/hive/branches/branch-0.3/ql/src/test/queries/clientpositive/input4_limit.q Sun
Apr 19 00:54:37 2009
@@ -0,0 +1,5 @@
+explain
+select * from src sort by key limit 10;
+
+
+select * from src sort by key limit 10;

Added: hadoop/hive/branches/branch-0.3/ql/src/test/results/clientpositive/input4_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/test/results/clientpositive/input4_limit.q.out?rev=766409&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/test/results/clientpositive/input4_limit.q.out
(added)
+++ hadoop/hive/branches/branch-0.3/ql/src/test/results/clientpositive/input4_limit.q.out
Sun Apr 19 00:54:37 2009
@@ -0,0 +1,81 @@
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_COLREF key)))
(TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src 
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: string
+                sort order: +
+                tag: -1
+                value expressions:
+                      expr: 0
+                      type: string
+                      expr: 1
+                      type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        /data/users/zshao/tools/deploy-branch-0.3-apache-hive/build/ql/tmp/350663458/433602273.10002

+          Reduce Output Operator
+            key expressions:
+                  expr: 0
+                  type: string
+            sort order: +
+            tag: -1
+            value expressions:
+                  expr: 0
+                  type: string
+                  expr: 1
+                  type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 10
+
+
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+100	val_100
+103	val_103
+103	val_103
+104	val_104
+104	val_104



Mime
View raw message