hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r752996 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/results/clientpositive/ ql/src/test/results/compiler/plan/
Date Thu, 12 Mar 2009 20:19:58 GMT
Author: zshao
Date: Thu Mar 12 20:19:57 2009
New Revision: 752996

URL: http://svn.apache.org/viewvc?rev=752996&view=rev
Log:
HIVE-339. Fix count distinct in 1 map-reduce job with map side aggregation. (Namit Jain via
zshao)

Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java
    hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Mar 12 20:19:57 2009
@@ -19,6 +19,9 @@
 
   BUG FIXES
 
+    HIVE-339. Fix count distinct in 1 map-reduce job with map side
+    aggregation. (Namit Jain via zshao)
+
     HIVE-271. Move test-udfs creation to build-common in order to
     fix broken 0.17 build. (athusoo)
     

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu
Mar 12 20:19:57 2009
@@ -407,10 +407,10 @@
     // Total number of input rows is needed for hash aggregation only
     if (hashAggr) {
       numRowsInput++;
-      // if hash aggregation is not behvaing properly, disable it
+      // if hash aggregation is not behaving properly, disable it
       if (numRowsInput == numRowsCompareHashAggr) {
         numRowsCompareHashAggr += groupbyMapAggrInterval;
-        // map-side aggregation should reduce the entries by atleast half
+        // map-side aggregation should reduce the entries by at-least half
         if ((numRowsHashTbl * 2) > numRowsInput) {
           LOG.warn("Disable Hash Aggr: #hash table = " + numRowsHashTbl + " #total = " +
numRowsInput);
           flush(true);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Thu Mar 12
20:19:57 2009
@@ -99,7 +99,7 @@
     return parentOperators;
   }
 
-  protected String id;
+  transient protected String id;
   protected T conf;
   protected boolean done;
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu
Mar 12 20:19:57 2009
@@ -1805,7 +1805,21 @@
    * @param input
    * @return
    * @throws SemanticException
-   */
+   *
+   * Generate a Group-By plan using 1 map-reduce job. 
+   * Spray by the group by key, and sort by the distinct key (if any), and 
+   * compute aggregates   * 
+   * The agggregation evaluation functions are as follows:
+   *   Partitioning Key: 
+   *      grouping key
+   *
+   *   Sorting Key: 
+   *      grouping key if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   * 
+   *   Reducer: iterate/merge
+   *   (mode = COMPLETE)
+   **/
   @SuppressWarnings({ "unused", "nls" })
   private Operator genGroupByPlan1MR(String dest, QB qb,
       Operator input) throws SemanticException {
@@ -1839,6 +1853,36 @@
    * @param input
    * @return
    * @throws SemanticException
+   *
+   * Generate a Group-By plan using a 2 map-reduce jobs. 
+   * Spray by the grouping key and distinct key (or a random number, if no distinct is 
+   * present) in hope of getting a uniform distribution, and compute partial aggregates 
+   * grouped by the reduction key (grouping key + distinct key).
+   * Evaluate partial aggregates first, and spray by the grouping key to compute actual
+   * aggregates in the second phase.
+   * The agggregation evaluation functions are as follows:
+   *   Partitioning Key: 
+   *      random() if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   *
+   *   Sorting Key: 
+   *      grouping key if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   * 
+   *   Reducer: iterate/terminatePartial
+   *   (mode = PARTIAL1)
+   * 
+   *   STAGE 2
+   *
+   *   Partitioning Key: 
+   *      grouping key 
+   *
+   *   Sorting Key: 
+   *      grouping key if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   *
+   *   Reducer: merge/terminate
+   *   (mode = FINAL)
    */
   @SuppressWarnings("nls")
   private Operator genGroupByPlan2MR(String dest, QB qb,
@@ -1883,11 +1927,26 @@
   }
 
   /**
-   * Generate a Group-By plan using a 2 map-reduce jobs. First perform a map
-   * side partial aggregation (to reduce the amount of data). Then spray by
-   * the distinct key (or a random number) in hope of getting a uniform 
-   * distribution, and compute partial aggregates grouped by that distinct key.
-   * Evaluate partial aggregates first, followed by actual aggregates.
+   * Generate a Group-By plan using 1 map-reduce job. 
+   * First perform a map-side partial aggregation (to reduce the amount of data), at this
+   * point of time, we may turn off map-side partial aggregation based on its performance.

+   * Then spray by the group by key, and sort by the distinct key (if any), and 
+   * compute aggregates based on actual aggregates
+   * 
+   * The agggregation evaluation functions are as follows:
+   *   Mapper: iterate/terminatePartial      
+   *   (mode = HASH)
+   *
+   *   Partitioning Key: 
+   *      grouping key
+   *
+   *   Sorting Key: 
+   *      grouping key if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   * 
+   *   Reducer: iterate/terminate if DISTINCT
+   *            merge/terminate if NO DISTINCT
+   *   (mode = MERGEPARTIAL)
    */
   @SuppressWarnings("nls")
   private Operator genGroupByPlanMapAggr1MR(String dest, QB qb, 
@@ -1910,15 +1969,51 @@
     Operator reduceSinkOperatorInfo = 
       genGroupByPlanReduceSinkOperator(qb, dest, groupByOperatorInfo, getGroupByForClause(parseInfo,
dest).size(), numReducers, true);
     
-    return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.FINAL);
+    // This is a 1-stage map-reduce processing of the groupby. Tha map-side aggregates was
just used to
+    // reduce output data. In case of distincts, partial results are not used, and so iterate
is again
+    // invoked on the reducer. In case of non-distincts, partial results are used, and merge
is invoked
+    // on the reducer.
+    return genGroupByPlanGroupByOperator1(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.MERGEPARTIAL);
   }
 
   /**
-   * Generate a Group-By plan using a 2 map-reduce jobs. First perform a map
-   * side partial aggregation (to reduce the amount of data). Then spray by
-   * the distinct key (or a random number) in hope of getting a uniform 
-   * distribution, and compute partial aggregates grouped by that distinct key.
-   * Evaluate partial aggregates first, followed by actual aggregates.
+   * Generate a Group-By plan using a 2 map-reduce jobs. 
+   * However, only 1 group-by plan is generated if the query involves no grouping key and
+   * no distincts. In that case, the plan is same as generated by genGroupByPlanMapAggr1MR.
+   * Otherwise, the following plan is generated:
+   * First perform a map side partial aggregation (to reduce the amount of data). Then 
+   * spray by the grouping key and distinct key (or a random number, if no distinct is 
+   * present) in hope of getting a uniform distribution, and compute partial aggregates 
+   * grouped by the reduction key (grouping key + distinct key).
+   * Evaluate partial aggregates first, and spray by the grouping key to compute actual
+   * aggregates in the second phase.
+   * The agggregation evaluation functions are as follows:
+   *   Mapper: iterate/terminatePartial      
+   *   (mode = HASH)
+   * 
+   *   Partitioning Key: 
+   *      random() if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   *
+   *   Sorting Key: 
+   *      grouping key if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   * 
+   *   Reducer: iterate/terminatePartial if DISTINCT
+   *            merge/terminatePartial if NO DISTINCT
+   *   (mode = PARTIAL2)
+   * 
+   *   STAGE 2
+   *
+   *   Partitioining Key: 
+   *      grouping key 
+   *
+   *   Sorting Key: 
+   *      grouping key if no DISTINCT
+   *      grouping + distinct key if DISTINCT
+   *
+   *   Reducer: merge/terminate
+   *   (mode = FINAL)
    */
   @SuppressWarnings("nls")
   private Operator genGroupByPlanMapAggr2MR(String dest, QB qb, 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java Thu Mar
12 20:19:57 2009
@@ -28,7 +28,7 @@
    *  HASH: the same as PARTIAL1 but use hash-table-based aggregation  
    */
   private static final long serialVersionUID = 1L;
-  public static enum Mode { COMPLETE, PARTIAL1, PARTIAL2, FINAL, HASH };
+  public static enum Mode { COMPLETE, PARTIAL1, PARTIAL2, FINAL, HASH, MERGEPARTIAL };
   private Mode mode;
   private java.util.ArrayList<exprNodeDesc> keys;
   private java.util.ArrayList<org.apache.hadoop.hive.ql.plan.aggregationDesc> aggregators;
@@ -63,6 +63,8 @@
       return "hash";
     case FINAL:
       return "final";
+    case MERGEPARTIAL:
+      return "mergepartial";
     }
   
     return "unknown";

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
Files hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out (original)
and hadoop/hive/trunk/ql/src/test/results/clientpositive/binarysortable_1.q.out Thu Mar 12
20:19:57 2009 differ

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out Thu Mar 12 20:19:57
2009
@@ -37,7 +37,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0
@@ -56,7 +56,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/262628633/486992722.10001 
+        /data/users/njain/hive1/hive/build/ql/tmp/746802126/1233171241.10001 
           Reduce Output Operator
             sort order: 
             tag: -1

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map.q.out Thu Mar 12 20:19:57
2009
@@ -36,7 +36,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_nomap.q.out Thu Mar
12 20:19:57 2009
@@ -36,7 +36,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out Thu Mar 12 20:19:57
2009
@@ -36,7 +36,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map.q.out Thu Mar 12 20:19:57
2009
@@ -40,11 +40,11 @@
         Group By Operator
           aggregations:
                 expr: sum(VALUE.0)
-                expr: count(VALUE.1)
+                expr: count(DISTINCT KEY.1)
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map.q.out Thu Mar 12 20:19:57
2009
@@ -47,10 +47,10 @@
           aggregations:
                 expr: sum(VALUE.0)
                 expr: max(VALUE.1)
-                expr: avg(VALUE.2)
+                expr: avg(DISTINCT UDFToDouble(KEY.0))
                 expr: min(VALUE.3)
                 expr: avg(VALUE.4)
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out Thu Mar 12 20:19:57
2009
@@ -25,7 +25,7 @@
         Group By Operator
           aggregations:
                 expr: count(VALUE.0)
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out Thu Mar 12 20:19:57
2009
@@ -28,7 +28,7 @@
         Group By Operator
           aggregations:
                 expr: sum(VALUE.0)
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map.q.out Thu Mar 12 20:19:57
2009
@@ -33,7 +33,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out Thu Mar 12 20:19:57
2009
@@ -52,7 +52,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0
@@ -87,7 +87,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/123213583/1044345267.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/272279570/1527190226.10002 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -107,7 +107,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out Thu Mar 12 20:19:57
2009
@@ -54,11 +54,11 @@
       Reduce Operator Tree:
         Group By Operator
           aggregations:
-                expr: count(VALUE.0)
+                expr: count(DISTINCT KEY.1)
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0
@@ -93,7 +93,7 @@
   Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
-        /data/users/njain/hive1/hive/build/ql/tmp/1475395360/1425775.10002 
+        /data/users/njain/hive1/hive/build/ql/tmp/1165802094/307152394.10002 
           Reduce Output Operator
             key expressions:
                   expr: 0
@@ -111,11 +111,11 @@
       Reduce Operator Tree:
         Group By Operator
           aggregations:
-                expr: count(VALUE.0)
+                expr: count(DISTINCT KEY.1)
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out Thu Mar 12 20:19:57
2009
@@ -39,11 +39,11 @@
       Reduce Operator Tree:
         Group By Operator
           aggregations:
-                expr: count(VALUE.0)
+                expr: count(DISTINCT KEY.1)
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0
@@ -167,7 +167,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias1.q.out Thu Mar 12 20:19:57
2009
@@ -44,7 +44,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: '1234'

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/notable_alias2.q.out Thu Mar 12 20:19:57
2009
@@ -44,7 +44,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: '1234'

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/subq2.q.out Thu Mar 12 20:19:57 2009
@@ -40,7 +40,7 @@
           keys:
                 expr: KEY.0
                 type: string
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf3.q.out Thu Mar 12 20:19:57 2009
@@ -41,7 +41,7 @@
                 expr: count(VALUE.2)
                 expr: sum(VALUE.3)
                 expr: min(VALUE.4)
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 2

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf8.q.out Thu Mar 12 20:19:57 2009
@@ -32,7 +32,7 @@
                 expr: sum(VALUE.0)
                 expr: count(VALUE.1)
                 expr: avg(VALUE.2)
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 2

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/union2.q.out Thu Mar 12 20:19:57
2009
@@ -47,7 +47,7 @@
         Group By Operator
           aggregations:
                 expr: count(VALUE.0)
-          mode: final
+          mode: mergepartial
           Select Operator
             expressions:
                   expr: 0

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml Thu Mar 12 20:19:57
2009
@@ -31,7 +31,7 @@
              <boolean>true</boolean> 
             </void> 
             <void property="sourceDir"> 
-             <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/1045332593/216206529.10000.insclause-0</string>

+             <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/434688883/186307528.10000.insclause-0</string>

             </void> 
             <void property="table"> 
              <object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">

@@ -584,7 +584,7 @@
                  <int>1</int> 
                 </void> 
                 <void property="dirName"> 
-                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/1045332593/216206529.10000.insclause-0</string>

+                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/434688883/186307528.10000.insclause-0</string>

                 </void> 
                 <void property="tableInfo"> 
                  <object idref="tableDesc0"/> 
@@ -739,7 +739,7 @@
         </void> 
         <void property="mode"> 
          <object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">

-          <string>FINAL</string> 
+          <string>MERGEPARTIAL</string> 
          </object> 
         </void> 
        </object> 

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml Thu Mar 12 20:19:57
2009
@@ -711,7 +711,7 @@
               <void property="conf"> 
                <object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc"> 
                 <void property="dirName"> 
-                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/717653767/1509670805.10001.insclause-0</string>

+                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/10857281/717275265.10001.insclause-0</string>

                 </void> 
                 <void property="tableInfo"> 
                  <object class="org.apache.hadoop.hive.ql.plan.tableDesc"> 
@@ -920,7 +920,7 @@
            <string>merge</string> 
           </void> 
           <void method="add"> 
-           <string>merge</string> 
+           <string>iterate</string> 
           </void> 
          </object> 
         </void> 
@@ -952,15 +952,18 @@
             <void property="aggregationClass"> 
              <class>org.apache.hadoop.hive.ql.udf.UDAFCount</class> 
             </void> 
+            <void property="distinct"> 
+             <boolean>true</boolean> 
+            </void> 
             <void property="parameters"> 
              <object class="java.util.ArrayList"> 
               <void method="add"> 
                <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
                 <void property="column"> 
-                 <string>VALUE.1</string> 
+                 <string>KEY.1</string> 
                 </void> 
                 <void property="typeInfo"> 
-                 <object idref="PrimitiveTypeInfo2"/> 
+                 <object idref="PrimitiveTypeInfo0"/> 
                 </void> 
                </object> 
               </void> 
@@ -996,7 +999,7 @@
         </void> 
         <void property="mode"> 
          <object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">

-          <string>FINAL</string> 
+          <string>MERGEPARTIAL</string> 
          </object> 
         </void> 
        </object> 

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml Thu Mar 12 20:19:57
2009
@@ -997,7 +997,7 @@
               <void property="conf"> 
                <object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc"> 
                 <void property="dirName"> 
-                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/334497369.10001.insclause-0</string>

+                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/150776966.10001.insclause-0</string>

                 </void> 
                 <void property="tableInfo"> 
                  <object class="org.apache.hadoop.hive.ql.plan.tableDesc"> 
@@ -1188,7 +1188,7 @@
            <string>merge</string> 
           </void> 
           <void method="add"> 
-           <string>merge</string> 
+           <string>iterate</string> 
           </void> 
           <void method="add"> 
            <string>merge</string> 
@@ -1247,15 +1247,42 @@
             <void property="aggregationClass"> 
              <class>org.apache.hadoop.hive.ql.udf.UDAFAvg</class> 
             </void> 
+            <void property="distinct"> 
+             <boolean>true</boolean> 
+            </void> 
             <void property="parameters"> 
              <object class="java.util.ArrayList"> 
               <void method="add"> 
-               <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
-                <void property="column"> 
-                 <string>VALUE.2</string> 
+               <object class="org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc"> 
+                <void property="UDFClass"> 
+                 <class>org.apache.hadoop.hive.ql.udf.UDFToDouble</class> 
+                </void> 
+                <void property="UDFMethod"> 
+                 <object class="org.apache.hadoop.hive.ql.udf.UDFToDouble" method="getMethod">

+                  <string>evaluate</string> 
+                  <array class="java.lang.Class" length="1"> 
+                   <void index="0"> 
+                    <class>java.lang.String</class> 
+                   </void> 
+                  </array> 
+                 </object> 
+                </void> 
+                <void property="children"> 
+                 <object class="java.util.ArrayList"> 
+                  <void method="add"> 
+                   <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc">

+                    <void property="column"> 
+                     <string>KEY.0</string> 
+                    </void> 
+                    <void property="typeInfo"> 
+                     <object idref="PrimitiveTypeInfo0"/> 
+                    </void> 
+                   </object> 
+                  </void> 
+                 </object> 
                 </void> 
                 <void property="typeInfo"> 
-                 <object idref="PrimitiveTypeInfo0"/> 
+                 <object idref="PrimitiveTypeInfo1"/> 
                 </void> 
                </object> 
               </void> 
@@ -1331,7 +1358,7 @@
         </void> 
         <void property="mode"> 
          <object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">

-          <string>FINAL</string> 
+          <string>MERGEPARTIAL</string> 
          </object> 
         </void> 
        </object> 

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml Thu Mar 12 20:19:57
2009
@@ -457,7 +457,7 @@
               <void property="conf"> 
                <object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc"> 
                 <void property="dirName"> 
-                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/69474526.10001.insclause-0</string>

+                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/57949151.10001.insclause-0</string>

                 </void> 
                 <void property="tableInfo"> 
                  <object class="org.apache.hadoop.hive.ql.plan.tableDesc"> 
@@ -584,7 +584,7 @@
         </void> 
         <void property="mode"> 
          <object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">

-          <string>FINAL</string> 
+          <string>MERGEPARTIAL</string> 
          </object> 
         </void> 
        </object> 

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml Thu Mar 12 20:19:57
2009
@@ -479,7 +479,7 @@
               <void property="conf"> 
                <object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc"> 
                 <void property="dirName"> 
-                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/1489470041/391886937.10001.insclause-0</string>

+                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/979288592/53411655.10001.insclause-0</string>

                 </void> 
                 <void property="tableInfo"> 
                  <object class="org.apache.hadoop.hive.ql.plan.tableDesc"> 
@@ -656,7 +656,7 @@
         </void> 
         <void property="mode"> 
          <object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">

-          <string>FINAL</string> 
+          <string>MERGEPARTIAL</string> 
          </object> 
         </void> 
        </object> 

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml Thu Mar 12 20:19:57
2009
@@ -457,7 +457,7 @@
               <void property="conf"> 
                <object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc"> 
                 <void property="dirName"> 
-                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/889541974.10001.insclause-0</string>

+                 <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/272705795.10001.insclause-0</string>

                 </void> 
                 <void property="tableInfo"> 
                  <object class="org.apache.hadoop.hive.ql.plan.tableDesc"> 
@@ -584,7 +584,7 @@
         </void> 
         <void property="mode"> 
          <object class="org.apache.hadoop.hive.ql.plan.groupByDesc$Mode" method="valueOf">

-          <string>FINAL</string> 
+          <string>MERGEPARTIAL</string> 
          </object> 
         </void> 
        </object> 

Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml?rev=752996&r1=752995&r2=752996&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml Thu Mar 12 20:19:57 2009
@@ -28,7 +28,7 @@
              <boolean>true</boolean> 
             </void> 
             <void property="sourceDir"> 
-             <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/752613561/196516335.10000.insclause-0</string>

+             <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/672033374/77772175.10000.insclause-0</string>

             </void> 
             <void property="targetDir"> 
              <string>../build/ql/test/data/warehouse/union.out</string> 
@@ -82,7 +82,7 @@
                                <int>1</int> 
                               </void> 
                               <void property="dirName"> 
-                               <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/752613561/196516335.10000.insclause-0</string>

+                               <string>/data/users/njain/hive1/hive/ql/../build/ql/tmp/672033374/77772175.10000.insclause-0</string>

                               </void> 
                               <void property="tableInfo"> 
                                <object class="org.apache.hadoop.hive.ql.plan.tableDesc">




Mime
View raw message