hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [24/51] [abbrv] hive git commit: HIVE-11923: allow qtests to run via a single client session for tez and llap (Siddharth Seth reviewed by Prasanth Jayachandran)
Date Tue, 13 Oct 2015 00:12:49 GMT
HIVE-11923: allow qtests to run via a single client session for tez and llap (Siddharth Seth
reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a3e7457
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a3e7457
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a3e7457

Branch: refs/heads/master
Commit: 7a3e74579f33ef96abcdf7f45c507c6d47b912c7
Parents: bf8acfb
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Mon Sep 28 19:40:05 2015 -0500
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Mon Sep 28 19:40:05 2015 -0500

----------------------------------------------------------------------
 data/conf/hive-log4j2.xml                       | 27 ++++++++++++++++++++
 data/conf/llap/tez-site.xml                     |  6 +++++
 data/conf/tez/tez-site.xml                      |  6 +++++
 .../org/apache/hadoop/hive/ql/QTestUtil.java    | 16 ++++++++++++
 4 files changed, 55 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
index 48a584f..860d756 100644
--- a/data/conf/hive-log4j2.xml
+++ b/data/conf/hive-log4j2.xml
@@ -72,6 +72,33 @@
       <AppenderRef ref="EventCounter" />
     </Root>
 
+    <Logger name="org.apache.hadoop.ipc" level="WARN">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.hadoop.security" level="INFO">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.hadoop.hdfs" level="INFO">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.hadoop.hdfs.server" level="WARN">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.hadoop.metrics2" level="INFO">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.mortbay" level="INFO">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.hadoop.yarn" level="INFO">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.hadoop.yarn.server" level="WARN">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.tez" level="INFO">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
     <Logger name="org.apache.hadoop.conf.Configuration" level="ERROR">
       <AppenderRef ref="${sys:hive.root.logger}"/>
     </Logger>

http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/llap/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml
new file mode 100644
index 0000000..940f390
--- /dev/null
+++ b/data/conf/llap/tez-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+  <property>
+    <name>tez.am.dag.scheduler.class</name>
+    <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/tez/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/tez/tez-site.xml b/data/conf/tez/tez-site.xml
new file mode 100644
index 0000000..940f390
--- /dev/null
+++ b/data/conf/tez/tez-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+  <property>
+    <name>tez.am.dag.scheduler.class</name>
+    <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 4cab161..477a08d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -81,6 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
 import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
 import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -158,6 +159,7 @@ public class QTestUtil {
   private final boolean miniMr = false;
   private String hadoopVer = null;
   private QTestSetup setup = null;
+  private TezSessionState tezSessionState = null;
   private SparkSession sparkSession = null;
   private boolean isSessionStateStarted = false;
   private static final String javaVersion = getJavaVersion();
@@ -428,10 +430,16 @@ public class QTestUtil {
 
       String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
       if (clusterType == MiniClusterType.tez) {
+        if (confDir != null && !confDir.isEmpty()) {
+          conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+              + "/tez-site.xml"));
+        }
         mr = shims.getMiniTezCluster(conf, 4, uriString, false);
       } else if (clusterType == MiniClusterType.llap) {
         if (confDir != null && !confDir.isEmpty()) {
           conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+              + "/tez-site.xml"));
+          conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
               + "/llap-daemon-site.xml"));
         }
         mr = shims.getMiniTezCluster(conf, 2, uriString, true);
@@ -952,6 +960,10 @@ public class QTestUtil {
       sparkSession = oldSs.getSparkSession();
       ss.setSparkSession(sparkSession);
       oldSs.setSparkSession(null);
+      // Copy the tezSessionState from the old CliSessionState.
+      tezSessionState = oldSs.getTezSession();
+      ss.setTezSession(tezSessionState);
+      oldSs.setTezSession(null);
       oldSs.close();
     }
 
@@ -1016,6 +1028,10 @@ public class QTestUtil {
       sparkSession = oldSs.getSparkSession();
       ss.setSparkSession(sparkSession);
       oldSs.setSparkSession(null);
+      // Copy the tezSessionState from the old CliSessionState.
+      tezSessionState = oldSs.getTezSession();
+      ss.setTezSession(tezSessionState);
+      oldSs.setTezSession(null);
       oldSs.close();
     }
     if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {


Mime
View raw message