Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8F4BB200D34 for ; Fri, 20 Oct 2017 03:11:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8DD21160BEE; Fri, 20 Oct 2017 01:11:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AB764160BEC for ; Fri, 20 Oct 2017 03:11:36 +0200 (CEST) Received: (qmail 12465 invoked by uid 500); 20 Oct 2017 01:11:35 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 12320 invoked by uid 99); 20 Oct 2017 01:11:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Oct 2017 01:11:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id ACB34DFAB2; Fri, 20 Oct 2017 01:11:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hashutosh@apache.org To: commits@hive.apache.org Date: Fri, 20 Oct 2017 01:11:36 -0000 Message-Id: <57c0474c1b3548e68759bc19d88decd4@git.apache.org> In-Reply-To: <2b66999937c942d980769d001847ff36@git.apache.org> References: <2b66999937c942d980769d001847ff36@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/3] hive git commit: HIVE-17817 : Stabilize crossproduct warning message output order (Zoltan Haindrich via Ashutosh Chauhan) archived-at: Fri, 20 Oct 2017 01:11:37 -0000 HIVE-17817 : Stabilize crossproduct warning message output order (Zoltan Haindrich via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/077a9e5b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/077a9e5b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/077a9e5b Branch: refs/heads/master Commit: 077a9e5b4d6566744498468be0e028f93514eb61 Parents: 39e1636 Author: Zoltan Haindrich Authored: Thu Oct 19 17:19:09 2017 -0700 Committer: Ashutosh Chauhan Committed: Thu Oct 19 18:11:14 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/cli/TestTezPerfCliDriver.java | 14 +++++++++++++- .../hadoop/hive/cli/control/CorePerfCliDriver.java | 8 ++++++++ .../hive/ql/optimizer/physical/CrossProductCheck.java | 4 ++-- .../ql/optimizer/physical/SparkCrossProductCheck.java | 5 ++--- .../java/org/apache/hadoop/hive/ql/plan/TezWork.java | 6 +++--- ql/src/test/queries/clientpositive/perf/query28.q | 2 ++ .../results/clientpositive/perf/tez/query14.q.out | 2 +- 7 files changed, 31 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java index 0c9b2ba..69578fc 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestTezPerfCliDriver.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.cli; import java.io.File; +import java.util.Comparator; import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; @@ -37,7 +38,18 @@ public class TestTezPerfCliDriver { @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + List parameters = adapter.getParameters(); + parameters.sort(new C1()); + return parameters; + } + + static class C1 implements Comparator { + + @Override + public int compare(Object[] o1, Object[] o2) { + return o1[0].toString().compareTo(o2[0].toString()); + } + } @ClassRule http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index d80bd44..7cfead8 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -86,6 +86,14 @@ public class CorePerfCliDriver extends CliAdapter{ @Override public void setUp() { + try { + qt.clearPostTestEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception"); + } } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java index f5abaf1..4b35bb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java @@ -27,6 +27,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.TreeMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -180,8 +181,7 @@ public class CrossProductCheck implements PhysicalPlanResolver, Dispatcher { ReduceWork rWork = (ReduceWork) wrk; Operator reducer = ((ReduceWork)wrk).getReducer(); if ( reducer instanceof JoinOperator || reducer instanceof CommonMergeJoinOperator ) { - Map rsInfo = - new HashMap(); + Map rsInfo = new TreeMap(); for(Map.Entry e : rWork.getTagToInput().entrySet()) { rsInfo.putAll(getReducerInfo(tzWrk, rWork.getName(), e.getValue())); } http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java index f904451..7f3b1b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkCrossProductCheck.java @@ -36,11 +36,11 @@ import org.apache.hadoop.hive.ql.session.SessionState; import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.TreeMap; /** * Check each MapJoin and ShuffleJoin Operator to see if they are performing a cross product. @@ -92,8 +92,7 @@ public class SparkCrossProductCheck implements PhysicalPlanResolver, Dispatcher for (ReduceWork reduceWork : sparkWork.getAllReduceWork()) { Operator reducer = reduceWork.getReducer(); if (reducer instanceof JoinOperator || reducer instanceof CommonMergeJoinOperator) { - Map rsInfo = - new HashMap(); + Map rsInfo = new TreeMap(); for (BaseWork parent : sparkWork.getParents(reduceWork)) { rsInfo.putAll(new CrossProductCheck.ExtractReduceSinkInfo(null).analyze(parent)); } http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java index a037ea3..18ff13b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java @@ -19,12 +19,12 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -74,8 +74,8 @@ public class TezWork extends AbstractOperatorDesc { private static final AtomicInteger counter = new AtomicInteger(1); private final String dagId; private final String queryName; - private final Set roots = new HashSet(); - private final Set leaves = new HashSet(); + private final Set roots = new LinkedHashSet(); + private final Set leaves = new LinkedHashSet(); private final Map> workGraph = new HashMap>(); private final Map> invertedWorkGraph = new HashMap>(); private final Map, TezEdgeProperty> edgeProperties = http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/test/queries/clientpositive/perf/query28.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query28.q b/ql/src/test/queries/clientpositive/perf/query28.q index 169653e..fc3c1b2d 100644 --- a/ql/src/test/queries/clientpositive/perf/query28.q +++ b/ql/src/test/queries/clientpositive/perf/query28.q @@ -55,3 +55,5 @@ from (select avg(ss_list_price) B1_LP limit 100; -- end query 1 in stream 0 using template query28.tpl + +set hive.optimize.metadataonly=false; http://git-wip-us.apache.org/repos/asf/hive/blob/077a9e5b/ql/src/test/results/clientpositive/perf/tez/query14.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out index 77959cb..b2a45f1 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out @@ -1,6 +1,6 @@ Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product +Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk