hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject [2/3] hive git commit: HIVE-17817 : Stabilize crossproduct warning message output order (Zoltan Haindrich via Ashutosh Chauhan)
Date Fri, 20 Oct 2017 01:11:36 GMT
HIVE-17817 : Stabilize crossproduct warning message output order (Zoltan Haindrich via Ashutosh
Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/077a9e5b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/077a9e5b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/077a9e5b

Branch: refs/heads/master
Commit: 077a9e5b4d6566744498468be0e028f93514eb61
Parents: 39e1636
Author: Zoltan Haindrich <kirk@rxd.hu>
Authored: Thu Oct 19 17:19:09 2017 -0700
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Oct 19 18:11:14 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/cli/TestTezPerfCliDriver.java  | 14 +++++++++++++-
 .../hadoop/hive/cli/control/CorePerfCliDriver.java    |  8 ++++++++
 .../hive/ql/optimizer/physical/CrossProductCheck.java |  4 ++--
 .../ql/optimizer/physical/SparkCrossProductCheck.java |  5 ++---
 .../java/org/apache/hadoop/hive/ql/plan/TezWork.java  |  6 +++---
 ql/src/test/queries/clientpositive/perf/query28.q     |  2 ++
 .../results/clientpositive/perf/tez/query14.q.out     |  2 +-
 7 files changed, 31 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java
b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java
index 0c9b2ba..69578fc 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.cli;
 
 import java.io.File;
+import java.util.Comparator;
 import java.util.List;
 
 import org.apache.hadoop.hive.cli.control.CliAdapter;
@@ -37,7 +38,18 @@ public class TestTezPerfCliDriver {
 
   @Parameters(name = "{0}")
   public static List<Object[]> getParameters() throws Exception {
-    return adapter.getParameters();
+    List<Object[]> parameters = adapter.getParameters();
+    parameters.sort(new C1());
+    return parameters;
+  }
+
+  static class C1 implements Comparator<Object[]> {
+
+    @Override
+    public int compare(Object[] o1, Object[] o2) {
+      return o1[0].toString().compareTo(o2[0].toString());
+    }
+
   }
 
   @ClassRule

http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index d80bd44..7cfead8 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -86,6 +86,14 @@ public class CorePerfCliDriver extends CliAdapter{
 
   @Override
   public void setUp() {
+    try {
+      qt.clearPostTestEffects();
+    } catch (Exception e) {
+      System.err.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.err.flush();
+      fail("Unexpected exception");
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java
index f5abaf1..4b35bb6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java
@@ -27,6 +27,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Stack;
+import java.util.TreeMap;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -180,8 +181,7 @@ public class CrossProductCheck implements PhysicalPlanResolver, Dispatcher
{
       ReduceWork rWork = (ReduceWork) wrk;
       Operator<? extends OperatorDesc> reducer = ((ReduceWork)wrk).getReducer();
       if ( reducer instanceof JoinOperator || reducer instanceof CommonMergeJoinOperator
) {
-        Map<Integer, ExtractReduceSinkInfo.Info> rsInfo =
-            new HashMap<Integer, ExtractReduceSinkInfo.Info>();
+        Map<Integer, ExtractReduceSinkInfo.Info> rsInfo = new TreeMap<Integer, ExtractReduceSinkInfo.Info>();
         for(Map.Entry<Integer, String> e : rWork.getTagToInput().entrySet()) {
           rsInfo.putAll(getReducerInfo(tzWrk, rWork.getName(), e.getValue()));
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java
index f904451..7f3b1b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java
@@ -36,11 +36,11 @@ import org.apache.hadoop.hive.ql.session.SessionState;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Stack;
+import java.util.TreeMap;
 
 /**
  * Check each MapJoin and ShuffleJoin Operator to see if they are performing a cross product.
@@ -92,8 +92,7 @@ public class SparkCrossProductCheck implements PhysicalPlanResolver, Dispatcher
     for (ReduceWork reduceWork : sparkWork.getAllReduceWork()) {
       Operator<? extends OperatorDesc> reducer = reduceWork.getReducer();
       if (reducer instanceof JoinOperator || reducer instanceof CommonMergeJoinOperator)
{
-        Map<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info> rsInfo =
-            new HashMap<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info>();
+        Map<Integer, CrossProductCheck.ExtractReduceSinkInfo.Info> rsInfo = new TreeMap<Integer,
CrossProductCheck.ExtractReduceSinkInfo.Info>();
         for (BaseWork parent : sparkWork.getParents(reduceWork)) {
           rsInfo.putAll(new CrossProductCheck.ExtractReduceSinkInfo(null).analyze(parent));
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
index a037ea3..18ff13b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
@@ -19,12 +19,12 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -74,8 +74,8 @@ public class TezWork extends AbstractOperatorDesc {
   private static final AtomicInteger counter = new AtomicInteger(1);
   private final String dagId;
   private final String queryName;
-  private final Set<BaseWork> roots = new HashSet<BaseWork>();
-  private final Set<BaseWork> leaves = new HashSet<BaseWork>();
+  private final Set<BaseWork> roots = new LinkedHashSet<BaseWork>();
+  private final Set<BaseWork> leaves = new LinkedHashSet<BaseWork>();
   private final Map<BaseWork, List<BaseWork>> workGraph = new HashMap<BaseWork,
List<BaseWork>>();
   private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new HashMap<BaseWork,
List<BaseWork>>();
   private final Map<Pair<BaseWork, BaseWork>, TezEdgeProperty> edgeProperties
=

http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/test/queries/clientpositive/perf/query28.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/perf/query28.q b/ql/src/test/queries/clientpositive/perf/query28.q
index 169653e..fc3c1b2d 100644
--- a/ql/src/test/queries/clientpositive/perf/query28.q
+++ b/ql/src/test/queries/clientpositive/perf/query28.q
@@ -55,3 +55,5 @@ from (select avg(ss_list_price) B1_LP
 limit 100;
 
 -- end query 1 in stream 0 using template query28.tpl
+
+set hive.optimize.metadataonly=false;

http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
index 77959cb..b2a45f1 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
@@ -1,6 +1,6 @@
 Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer
5' is a cross product
-Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer
16' is a cross product
 Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer
12' is a cross product
+Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer
16' is a cross product
 PREHOOK: query: explain
 with  cross_items as
  (select i_item_sk ss_item_sk


Mime
View raw message