hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [3/5] hive git commit: HIVE-14830. Move a majority of the MiniLlapCliDriver tests to use an inline AM. (Siddharth Seth, reviewed by Prasanth Jayachandran)
Date Mon, 03 Oct 2016 23:45:00 GMT
HIVE-14830. Move a majority of the MiniLlapCliDriver tests to use an inline AM. (Siddharth
Seth, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0a4b3d8f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0a4b3d8f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0a4b3d8f

Branch: refs/heads/hive-14535
Commit: 0a4b3d8ff673f6f6670293a7491873c229cb0f40
Parents: b86342f
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Oct 3 13:05:34 2016 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Oct 3 13:05:34 2016 -0700

----------------------------------------------------------------------
 .../hive/cli/TestMiniLlapLocalCliDriver.java    |  62 ++++++++++
 .../test/resources/testconfiguration.properties | 117 ++++++++++---------
 .../hadoop/hive/cli/control/CliConfigs.java     |  29 +++++
 .../hive/ql/optimizer/physical/LlapDecider.java |   1 +
 4 files changed, 152 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
new file mode 100644
index 0000000..5b1caf6
--- /dev/null
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.hadoop.hive.cli.control.CliAdapter;
+import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestMiniLlapLocalCliDriver {
+
+  static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter();
+
+  @Parameters(name = "{0}")
+  public static List<Object[]> getParameters() throws Exception {
+    return adapter.getParameters();
+  }
+
+  @ClassRule
+  public static TestRule cliClassRule = adapter.buildClassRule();
+
+  @Rule
+  public TestRule cliTestRule = adapter.buildTestRule();
+
+  private String name;
+  private File qfile;
+
+  public TestMiniLlapLocalCliDriver(String name, File qfile) {
+    this.name = name;
+    this.qfile = qfile;
+  }
+
+  @Test
+  public void testCliDriver() throws Exception {
+    adapter.runTest(name, qfile);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e810a58..fbba0cd 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -54,8 +54,40 @@ minitez.query.files=explainuser_3.q,\
   stats_filemetadata.q,\
   tez_union_with_udf.q
 
+
 minillap.shared.query.files=acid_globallimit.q,\
-  alter_merge_2_orc.q,\
+  insert_into1.q,\
+  insert_into2.q,\
+  insert_values_orig_table.,\
+  llapdecider.q,\
+  mapreduce1.q,\
+  mapreduce2.q,\
+  orc_merge1.q,\
+  orc_merge10.q,\
+  orc_merge2.q,\
+  orc_merge3.q,\
+  orc_merge4.q,\
+  orc_merge_diff_fs.q,\
+  script_env_var1.q,\
+  script_env_var2.q,\
+  script_pipe.q,\
+  tez_union_view.q,\
+  transform1.q,\
+  transform2.q,\
+  transform_ppr1.q,\
+  transform_ppr2.q,\
+  unionDistinct_1.q,\
+  union_type_chk.q,\
+  orc_ppd_basic.q,\
+  orc_ppd_schema_evol_3a.q,\
+  cte_2.q,\
+  cte_4.q,\
+  llap_nullscan.q,\
+  dynamic_partition_pruning_2.q,\
+  tez_union_dynamic_partition.q,\
+  union_fast_stats.q
+
+minillaplocal.shared.query.files=alter_merge_2_orc.q,\
   alter_merge_orc.q,\
   alter_merge_stats_orc.q,\
   auto_join0.q,\
@@ -111,9 +143,7 @@ minillap.shared.query.files=acid_globallimit.q,\
   cross_product_check_2.q,\
   ctas.q,\
   cte_1.q,\
-  cte_2.q,\
   cte_3.q,\
-  cte_4.q,\
   cte_5.q,\
   cte_mat_1.q,\
   cte_mat_2.q,\
@@ -131,7 +161,6 @@ minillap.shared.query.files=acid_globallimit.q,\
   delete_whole_partition.q,\
   disable_merge_for_bucketing.q,\
   dynamic_partition_pruning.q,\
-  dynamic_partition_pruning_2.q,\
   dynpart_sort_opt_vectorization.q,\
   dynpart_sort_optimization.q,\
   dynpart_sort_optimization2.q,\
@@ -145,13 +174,10 @@ minillap.shared.query.files=acid_globallimit.q,\
   having.q,\
   identity_project_remove_skip.q,\
   insert1.q,\
-  insert_into1.q,\
-  insert_into2.q,\
   insert_orig_table.q,\
   insert_update_delete.q,\
   insert_values_dynamic_partitioned.q,\
   insert_values_non_partitioned.q,\
-  insert_values_orig_table.,\
   insert_values_partitioned.q,\
   insert_values_tmp_table.q,\
   join0.q,\
@@ -159,8 +185,6 @@ minillap.shared.query.files=acid_globallimit.q,\
   join_nullsafe.q,\
   leftsemijoin.q,\
   limit_pushdown.q,\
-  llap_nullscan.q,\
-  llapdecider.q,\
   load_dyn_part1.q,\
   load_dyn_part2.q,\
   load_dyn_part3.q,\
@@ -168,8 +192,6 @@ minillap.shared.query.files=acid_globallimit.q,\
   mapjoin2.q,\
   mapjoin_decimal.q,\
   mapjoin_mapjoin.q,\
-  mapreduce1.q,\
-  mapreduce2.q,\
   merge1.q,\
   merge2.q,\
   mergejoin.q,\
@@ -180,36 +202,25 @@ minillap.shared.query.files=acid_globallimit.q,\
   nonmr_fetch_threshold.q,\
   optimize_nullscan.q,\
   orc_analyze.q,\
-  orc_merge1.q,\
-  orc_merge10.q,\
   orc_merge11.q,\
-  orc_merge2.q,\
-  orc_merge3.q,\
-  orc_merge4.q,\
   orc_merge5.q,\
   orc_merge6.q,\
   orc_merge7.q,\
   orc_merge8.q,\
   orc_merge9.q,\
-  orc_merge_diff_fs.q,\
   orc_merge_incompat1.q,\
   orc_merge_incompat2.q,\
   orc_merge_incompat3.q,\
-  orc_ppd_basic.q,\
   orc_ppd_schema_evol_1a.q,\
   orc_ppd_schema_evol_1b.q,\
   orc_ppd_schema_evol_2a.q,\
   orc_ppd_schema_evol_2b.q,\
-  orc_ppd_schema_evol_3a.q,\
   order_null.q,\
   parallel.q,\
   ptf.q,\
   ptf_matchpath.q,\
   ptf_streaming.q,\
   sample1.q,\
-  script_env_var1.q,\
-  script_env_var2.q,\
-  script_pipe.q,\
   selectDistinctStar.q,\
   select_dummy_source.q,\
   skewjoin.q,\
@@ -237,16 +248,10 @@ minillap.shared.query.files=acid_globallimit.q,\
   tez_union.q,\
   tez_union2.q,\
   tez_union_decimal.q,\
-  tez_union_dynamic_partition.q,\
   tez_union_group_by.q,\
   tez_union_multiinsert.q,\
-  tez_union_view.q,\
   tez_vector_dynpart_hashjoin_1.q,\
   tez_vector_dynpart_hashjoin_2.q,\
-  transform1.q,\
-  transform2.q,\
-  transform_ppr1.q,\
-  transform_ppr2.q,\
   union2.q,\
   union3.q,\
   union4.q,\
@@ -255,10 +260,7 @@ minillap.shared.query.files=acid_globallimit.q,\
   union7.q,\
   union8.q,\
   union9.q,\
-  unionDistinct_1.q,\
-  union_fast_stats.q,\
   union_stats.q,\
-  union_type_chk.q,\
   update_after_multiple_inserts.q,\
   update_all_non_partitioned.q,\
   update_all_partitioned.q,\
@@ -403,11 +405,35 @@ minillap.shared.query.files=acid_globallimit.q,\
   vectorized_timestamp_ints_casts.q
 
 minillap.query.files=acid_bucket_pruning.q,\
-  acid_vectorization_missing_cols.q,\
-  auto_sortmerge_join_16.q,\
-  bucket4.q,\
   bucket5.q,\
   bucket6.q,\
+  explainuser_2.q,\
+  empty_dir_in_table.q,\
+  llap_udf.q,\
+  llapdecider.q,\
+  reduce_deduplicate.q,\
+  remote_script.q,\
+  tez_aggr_part_stats.q,\
+  tez_union_view.q,\
+  file_with_header_footer.q,\
+  external_table_with_space_in_location_path.q,\
+  import_exported_table.q,\
+  orc_llap_counters.q,\
+  orc_llap_counters1.q,\
+  load_hdfs_file_with_space_in_the_name.q,\
+  orc_ppd_basic.q,\
+  schemeAuthority.q,\
+  schemeAuthority2.q,\
+  temp_table_external.q,\
+  table_nonprintable.q,\
+  llap_nullscan.q,\
+  dynamic_partition_pruning_2.q,\
+  tez_union_dynamic_partition.q,\
+  load_fs2.q
+
+minillaplocal.query.files=acid_vectorization_missing_cols.q,\
+  auto_sortmerge_join_16.q,\
+  bucket4.q,\
   bucket_many.q,\
   bucket_map_join_tez1.q,\
   bucket_map_join_tez2.q,\
@@ -418,18 +444,12 @@ minillap.query.files=acid_bucket_pruning.q,\
   constprog_dpp.q,\
   disable_merge_for_bucketing.q,\
   dynamic_partition_pruning.q,\
-  dynamic_partition_pruning_2.q,\
-  empty_dir_in_table.q,\
   exchgpartition2lel.q,\
   explainuser_1.q,\
-  explainuser_2.q,\
   explainuser_4.q,\
-  external_table_with_space_in_location_path.q,\
-  file_with_header_footer.q,\
   groupby2.q,\
   hybridgrace_hashjoin_1.q,\
   hybridgrace_hashjoin_2.q,\
-  import_exported_table.q,\
   infer_bucket_sort_bucketed_table.q,\
   input16_cc.q,\
   insert_dir_distcp.q,\
@@ -437,24 +457,14 @@ minillap.query.files=acid_bucket_pruning.q,\
   join_acid_non_acid.q,\
   leftsemijoin_mr.q,\
   list_bucket_dml_10.q,\
-  llap_nullscan.q,\
-  llap_udf.q,\
-  llapdecider.q,\
-  load_fs2.q,\
-  load_hdfs_file_with_space_in_the_name.q,\
   lvj_mapjoin.q,\
   mapjoin_decimal.q,\
   mergejoin_3way.q,\
   mrr.q,\
   non_native_window_udf.q,\
   orc_llap.q,\
-  orc_llap_counters.q,\
-  orc_llap_counters1.q,\
   orc_llap_nonvector.q,\
-  orc_ppd_basic.q,\
   quotedid_smb.q,\
-  reduce_deduplicate.q,\
-  remote_script.q,\
   schema_evol_orc_acid_part.q,\
   schema_evol_orc_acid_part_update.q,\
   schema_evol_orc_acid_table.q,\
@@ -484,12 +494,7 @@ minillap.query.files=acid_bucket_pruning.q,\
   schema_evol_text_vecrow_part_all_complex.q,\
   schema_evol_text_vecrow_part_all_primitive.q,\
   schema_evol_text_vecrow_table.q,\
-  schemeAuthority.q,\
-  schemeAuthority2.q,\
   smb_cache.q,\
-  table_nonprintable.q,\
-  temp_table_external.q,\
-  tez_aggr_part_stats.q,\
   tez_bmj_schema_evolution.q,\
   tez_dml.q,\
   tez_dynpart_hashjoin_1.q,\
@@ -510,10 +515,8 @@ minillap.query.files=acid_bucket_pruning.q,\
   tez_union.q,\
   tez_union2.q,\
   tez_union_decimal.q,\
-  tez_union_dynamic_partition.q,\
   tez_union_group_by.q,\
   tez_union_multiinsert.q,\
-  tez_union_view.q,\
   tez_vector_dynpart_hashjoin_1.q,\
   tez_vector_dynpart_hashjoin_2.q,\
   uber_reduce.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index ca72282..201c3d3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -47,6 +47,7 @@ public class CliConfigs {
         setQueryDir("ql/src/test/queries/clientpositive");
 
         excludesFrom(testConfigProps, "minillap.query.files");
+        excludesFrom(testConfigProps, "minillaplocal.query.files");
         excludesFrom(testConfigProps, "minimr.query.files");
         excludesFrom(testConfigProps, "minitez.query.files");
         excludesFrom(testConfigProps, "encrypted.query.files");
@@ -129,6 +130,7 @@ public class CliConfigs {
         setHiveConfDir("data/conf/tez");
         setClusterType(MiniClusterType.tez);
         setMetastoreType(MetastoreType.hbase);
+        setFsType(QTestUtil.FsType.hdfs);
       } catch (Exception e) {
         throw new RuntimeException("can't construct cliconfig", e);
       }
@@ -152,6 +154,33 @@ public class CliConfigs {
 
         setHiveConfDir("data/conf/llap");
         setClusterType(MiniClusterType.llap);
+        setMetastoreType(MetastoreType.sql);
+      } catch (Exception e) {
+        throw new RuntimeException("can't construct cliconfig", e);
+      }
+    }
+  }
+
+  public static class MiniLlapLocalCliConfig extends AbstractCliConfig {
+
+    public MiniLlapLocalCliConfig() {
+      super(CoreCliDriver.class);
+      try {
+        setQueryDir("ql/src/test/queries/clientpositive");
+
+        includesFrom(testConfigProps, "minillaplocal.query.files");
+        includesFrom(testConfigProps, "minillaplocal.shared.query.files");
+
+        setResultsDir("ql/src/test/results/clientpositive/llap");
+        setLogDir("itests/qtest/target/qfile-results/clientpositive");
+
+        setInitScript("q_test_init.sql");
+        setCleanupScript("q_test_cleanup.sql");
+
+        setHiveConfDir("data/conf/llap");
+        setClusterType(MiniClusterType.llap_local);
+        setMetastoreType(MetastoreType.sql);
+        setFsType(QTestUtil.FsType.local);
       } catch (Exception e) {
         throw new RuntimeException("can't construct cliconfig", e);
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index 1ab7137..f46e2d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -156,6 +156,7 @@ public class LlapDecider implements PhysicalPlanResolver {
         if (tezWork.getChildren(work).isEmpty()
             && work instanceof ReduceWork
             && ((ReduceWork) work).getNumReduceTasks() == 1) {
+          LOG.info("Converting work to uber: {}", work);
           work.setUberMode(true);
         }
       }


Mime
View raw message