Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E0274200BBB for ; Thu, 6 Oct 2016 02:18:35 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id DED8F160AF2; Thu, 6 Oct 2016 00:18:35 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D3DF9160AF3 for ; Thu, 6 Oct 2016 02:18:34 +0200 (CEST) Received: (qmail 93635 invoked by uid 500); 6 Oct 2016 00:18:31 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 92862 invoked by uid 99); 6 Oct 2016 00:18:31 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Oct 2016 00:18:31 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 642CEE0BD9; Thu, 6 Oct 2016 00:18:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: khorgath@apache.org To: commits@hive.apache.org Date: Thu, 06 Oct 2016 00:18:52 -0000 Message-Id: <2daf3b6d761a4ffc901c43137bd814c3@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [23/38] hive git commit: HIVE-14830. Move a majority of the MiniLlapCliDriver tests to use an inline AM. (Siddharth Seth, reviewed by Prasanth Jayachandran) archived-at: Thu, 06 Oct 2016 00:18:36 -0000 HIVE-14830. Move a majority of the MiniLlapCliDriver tests to use an inline AM. (Siddharth Seth, reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0a4b3d8f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0a4b3d8f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0a4b3d8f Branch: refs/heads/repl2 Commit: 0a4b3d8ff673f6f6670293a7491873c229cb0f40 Parents: b86342f Author: Siddharth Seth Authored: Mon Oct 3 13:05:34 2016 -0700 Committer: Siddharth Seth Committed: Mon Oct 3 13:05:34 2016 -0700 ---------------------------------------------------------------------- .../hive/cli/TestMiniLlapLocalCliDriver.java | 62 ++++++++++ .../test/resources/testconfiguration.properties | 117 ++++++++++--------- .../hadoop/hive/cli/control/CliConfigs.java | 29 +++++ .../hive/ql/optimizer/physical/LlapDecider.java | 1 + 4 files changed, 152 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java new file mode 100644 index 0000000..5b1caf6 --- /dev/null +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli; + +import java.io.File; +import java.util.List; + +import org.apache.hadoop.hive.cli.control.CliAdapter; +import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestMiniLlapLocalCliDriver { + + static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter(); + + @Parameters(name = "{0}") + public static List getParameters() throws Exception { + return adapter.getParameters(); + } + + @ClassRule + public static TestRule cliClassRule = adapter.buildClassRule(); + + @Rule + public TestRule cliTestRule = adapter.buildTestRule(); + + private String name; + private File qfile; + + public TestMiniLlapLocalCliDriver(String name, File qfile) { + this.name = name; + this.qfile = qfile; + } + + @Test + public void testCliDriver() throws Exception { + adapter.runTest(name, qfile); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index e810a58..fbba0cd 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -54,8 +54,40 @@ minitez.query.files=explainuser_3.q,\ stats_filemetadata.q,\ tez_union_with_udf.q + minillap.shared.query.files=acid_globallimit.q,\ - alter_merge_2_orc.q,\ + insert_into1.q,\ + insert_into2.q,\ + insert_values_orig_table.,\ + llapdecider.q,\ + mapreduce1.q,\ + mapreduce2.q,\ + orc_merge1.q,\ + orc_merge10.q,\ + orc_merge2.q,\ + orc_merge3.q,\ + orc_merge4.q,\ + orc_merge_diff_fs.q,\ + script_env_var1.q,\ + script_env_var2.q,\ + script_pipe.q,\ + tez_union_view.q,\ + transform1.q,\ + transform2.q,\ + transform_ppr1.q,\ + transform_ppr2.q,\ + unionDistinct_1.q,\ + union_type_chk.q,\ + orc_ppd_basic.q,\ + orc_ppd_schema_evol_3a.q,\ + cte_2.q,\ + cte_4.q,\ + llap_nullscan.q,\ + dynamic_partition_pruning_2.q,\ + tez_union_dynamic_partition.q,\ + union_fast_stats.q + +minillaplocal.shared.query.files=alter_merge_2_orc.q,\ alter_merge_orc.q,\ alter_merge_stats_orc.q,\ auto_join0.q,\ @@ -111,9 +143,7 @@ minillap.shared.query.files=acid_globallimit.q,\ cross_product_check_2.q,\ ctas.q,\ cte_1.q,\ - cte_2.q,\ cte_3.q,\ - cte_4.q,\ cte_5.q,\ cte_mat_1.q,\ cte_mat_2.q,\ @@ -131,7 +161,6 @@ minillap.shared.query.files=acid_globallimit.q,\ delete_whole_partition.q,\ disable_merge_for_bucketing.q,\ dynamic_partition_pruning.q,\ - dynamic_partition_pruning_2.q,\ dynpart_sort_opt_vectorization.q,\ dynpart_sort_optimization.q,\ dynpart_sort_optimization2.q,\ @@ -145,13 +174,10 @@ minillap.shared.query.files=acid_globallimit.q,\ having.q,\ identity_project_remove_skip.q,\ insert1.q,\ - insert_into1.q,\ - insert_into2.q,\ insert_orig_table.q,\ insert_update_delete.q,\ insert_values_dynamic_partitioned.q,\ insert_values_non_partitioned.q,\ - insert_values_orig_table.,\ insert_values_partitioned.q,\ insert_values_tmp_table.q,\ join0.q,\ @@ -159,8 +185,6 @@ minillap.shared.query.files=acid_globallimit.q,\ join_nullsafe.q,\ leftsemijoin.q,\ limit_pushdown.q,\ - llap_nullscan.q,\ - llapdecider.q,\ load_dyn_part1.q,\ load_dyn_part2.q,\ load_dyn_part3.q,\ @@ -168,8 +192,6 @@ minillap.shared.query.files=acid_globallimit.q,\ mapjoin2.q,\ mapjoin_decimal.q,\ mapjoin_mapjoin.q,\ - mapreduce1.q,\ - mapreduce2.q,\ merge1.q,\ merge2.q,\ mergejoin.q,\ @@ -180,36 +202,25 @@ minillap.shared.query.files=acid_globallimit.q,\ nonmr_fetch_threshold.q,\ optimize_nullscan.q,\ orc_analyze.q,\ - orc_merge1.q,\ - orc_merge10.q,\ orc_merge11.q,\ - orc_merge2.q,\ - orc_merge3.q,\ - orc_merge4.q,\ orc_merge5.q,\ orc_merge6.q,\ orc_merge7.q,\ orc_merge8.q,\ orc_merge9.q,\ - orc_merge_diff_fs.q,\ orc_merge_incompat1.q,\ orc_merge_incompat2.q,\ orc_merge_incompat3.q,\ - orc_ppd_basic.q,\ orc_ppd_schema_evol_1a.q,\ orc_ppd_schema_evol_1b.q,\ orc_ppd_schema_evol_2a.q,\ orc_ppd_schema_evol_2b.q,\ - orc_ppd_schema_evol_3a.q,\ order_null.q,\ parallel.q,\ ptf.q,\ ptf_matchpath.q,\ ptf_streaming.q,\ sample1.q,\ - script_env_var1.q,\ - script_env_var2.q,\ - script_pipe.q,\ selectDistinctStar.q,\ select_dummy_source.q,\ skewjoin.q,\ @@ -237,16 +248,10 @@ minillap.shared.query.files=acid_globallimit.q,\ tez_union.q,\ tez_union2.q,\ tez_union_decimal.q,\ - tez_union_dynamic_partition.q,\ tez_union_group_by.q,\ tez_union_multiinsert.q,\ - tez_union_view.q,\ tez_vector_dynpart_hashjoin_1.q,\ tez_vector_dynpart_hashjoin_2.q,\ - transform1.q,\ - transform2.q,\ - transform_ppr1.q,\ - transform_ppr2.q,\ union2.q,\ union3.q,\ union4.q,\ @@ -255,10 +260,7 @@ minillap.shared.query.files=acid_globallimit.q,\ union7.q,\ union8.q,\ union9.q,\ - unionDistinct_1.q,\ - union_fast_stats.q,\ union_stats.q,\ - union_type_chk.q,\ update_after_multiple_inserts.q,\ update_all_non_partitioned.q,\ update_all_partitioned.q,\ @@ -403,11 +405,35 @@ minillap.shared.query.files=acid_globallimit.q,\ vectorized_timestamp_ints_casts.q minillap.query.files=acid_bucket_pruning.q,\ - acid_vectorization_missing_cols.q,\ - auto_sortmerge_join_16.q,\ - bucket4.q,\ bucket5.q,\ bucket6.q,\ + explainuser_2.q,\ + empty_dir_in_table.q,\ + llap_udf.q,\ + llapdecider.q,\ + reduce_deduplicate.q,\ + remote_script.q,\ + tez_aggr_part_stats.q,\ + tez_union_view.q,\ + file_with_header_footer.q,\ + external_table_with_space_in_location_path.q,\ + import_exported_table.q,\ + orc_llap_counters.q,\ + orc_llap_counters1.q,\ + load_hdfs_file_with_space_in_the_name.q,\ + orc_ppd_basic.q,\ + schemeAuthority.q,\ + schemeAuthority2.q,\ + temp_table_external.q,\ + table_nonprintable.q,\ + llap_nullscan.q,\ + dynamic_partition_pruning_2.q,\ + tez_union_dynamic_partition.q,\ + load_fs2.q + +minillaplocal.query.files=acid_vectorization_missing_cols.q,\ + auto_sortmerge_join_16.q,\ + bucket4.q,\ bucket_many.q,\ bucket_map_join_tez1.q,\ bucket_map_join_tez2.q,\ @@ -418,18 +444,12 @@ minillap.query.files=acid_bucket_pruning.q,\ constprog_dpp.q,\ disable_merge_for_bucketing.q,\ dynamic_partition_pruning.q,\ - dynamic_partition_pruning_2.q,\ - empty_dir_in_table.q,\ exchgpartition2lel.q,\ explainuser_1.q,\ - explainuser_2.q,\ explainuser_4.q,\ - external_table_with_space_in_location_path.q,\ - file_with_header_footer.q,\ groupby2.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ - import_exported_table.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ insert_dir_distcp.q,\ @@ -437,24 +457,14 @@ minillap.query.files=acid_bucket_pruning.q,\ join_acid_non_acid.q,\ leftsemijoin_mr.q,\ list_bucket_dml_10.q,\ - llap_nullscan.q,\ - llap_udf.q,\ - llapdecider.q,\ - load_fs2.q,\ - load_hdfs_file_with_space_in_the_name.q,\ lvj_mapjoin.q,\ mapjoin_decimal.q,\ mergejoin_3way.q,\ mrr.q,\ non_native_window_udf.q,\ orc_llap.q,\ - orc_llap_counters.q,\ - orc_llap_counters1.q,\ orc_llap_nonvector.q,\ - orc_ppd_basic.q,\ quotedid_smb.q,\ - reduce_deduplicate.q,\ - remote_script.q,\ schema_evol_orc_acid_part.q,\ schema_evol_orc_acid_part_update.q,\ schema_evol_orc_acid_table.q,\ @@ -484,12 +494,7 @@ minillap.query.files=acid_bucket_pruning.q,\ schema_evol_text_vecrow_part_all_complex.q,\ schema_evol_text_vecrow_part_all_primitive.q,\ schema_evol_text_vecrow_table.q,\ - schemeAuthority.q,\ - schemeAuthority2.q,\ smb_cache.q,\ - table_nonprintable.q,\ - temp_table_external.q,\ - tez_aggr_part_stats.q,\ tez_bmj_schema_evolution.q,\ tez_dml.q,\ tez_dynpart_hashjoin_1.q,\ @@ -510,10 +515,8 @@ minillap.query.files=acid_bucket_pruning.q,\ tez_union.q,\ tez_union2.q,\ tez_union_decimal.q,\ - tez_union_dynamic_partition.q,\ tez_union_group_by.q,\ tez_union_multiinsert.q,\ - tez_union_view.q,\ tez_vector_dynpart_hashjoin_1.q,\ tez_vector_dynpart_hashjoin_2.q,\ uber_reduce.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index ca72282..201c3d3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -47,6 +47,7 @@ public class CliConfigs { setQueryDir("ql/src/test/queries/clientpositive"); excludesFrom(testConfigProps, "minillap.query.files"); + excludesFrom(testConfigProps, "minillaplocal.query.files"); excludesFrom(testConfigProps, "minimr.query.files"); excludesFrom(testConfigProps, "minitez.query.files"); excludesFrom(testConfigProps, "encrypted.query.files"); @@ -129,6 +130,7 @@ public class CliConfigs { setHiveConfDir("data/conf/tez"); setClusterType(MiniClusterType.tez); setMetastoreType(MetastoreType.hbase); + setFsType(QTestUtil.FsType.hdfs); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } @@ -152,6 +154,33 @@ public class CliConfigs { setHiveConfDir("data/conf/llap"); setClusterType(MiniClusterType.llap); + setMetastoreType(MetastoreType.sql); + } catch (Exception e) { + throw new RuntimeException("can't construct cliconfig", e); + } + } + } + + public static class MiniLlapLocalCliConfig extends AbstractCliConfig { + + public MiniLlapLocalCliConfig() { + super(CoreCliDriver.class); + try { + setQueryDir("ql/src/test/queries/clientpositive"); + + includesFrom(testConfigProps, "minillaplocal.query.files"); + includesFrom(testConfigProps, "minillaplocal.shared.query.files"); + + setResultsDir("ql/src/test/results/clientpositive/llap"); + setLogDir("itests/qtest/target/qfile-results/clientpositive"); + + setInitScript("q_test_init.sql"); + setCleanupScript("q_test_cleanup.sql"); + + setHiveConfDir("data/conf/llap"); + setClusterType(MiniClusterType.llap_local); + setMetastoreType(MetastoreType.sql); + setFsType(QTestUtil.FsType.local); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index 1ab7137..f46e2d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -156,6 +156,7 @@ public class LlapDecider implements PhysicalPlanResolver { if (tezWork.getChildren(work).isEmpty() && work instanceof ReduceWork && ((ReduceWork) work).getNumReduceTasks() == 1) { + LOG.info("Converting work to uber: {}", work); work.setUberMode(true); } }