hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1632705 - in /hive/branches/branch-0.14: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Date Sat, 18 Oct 2014 00:18:39 GMT
Author: gunther
Date: Sat Oct 18 00:18:38 2014
New Revision: 1632705

URL: http://svn.apache.org/r1632705
Log:
HIVE-8452: Cleanup handling of resource configuration for tez (Gunther Hagleitner, reviewed
by Vikram Dixit K)

Modified:
    hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java

Modified: hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1632705&r1=1632704&r2=1632705&view=diff
==============================================================================
--- hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/branch-0.14/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat
Oct 18 00:18:38 2014
@@ -958,6 +958,9 @@ public class HiveConf extends Configurat
 
     HIVETEZCONTAINERSIZE("hive.tez.container.size", -1,
         "By default Tez will spawn containers of the size of a mapper. This can be used to
overwrite."),
+    HIVETEZCPUVCORES("hive.tez.cpu.vcores", -1,
+        "By default Tez will ask for however many cpus map-reduce is configured to use per
container. "
+        +"This can be used to overwrite."),
     HIVETEZJAVAOPTS("hive.tez.java.opts", null,
         "By default Tez will use the Java options from map tasks. This can be used to overwrite."),
     HIVETEZLOGLEVEL("hive.tez.log.level", "INFO",

Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1632705&r1=1632704&r2=1632705&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Sat Oct 18 00:18:38 2014
@@ -431,8 +431,9 @@ public class DagUtils {
     int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0
?
       HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) :
       conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB);
-    int cpus = conf.getInt(MRJobConfig.MAP_CPU_VCORES,
-        MRJobConfig.DEFAULT_MAP_CPU_VCORES);
+    int cpus = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) > 0 ?
+      HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) : 
+      conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES);
     return Resource.newInstance(memory, cpus);
   }
 
@@ -452,17 +453,29 @@ public class DagUtils {
    */
   private String getContainerJavaOpts(Configuration conf) {
     String javaOpts = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZJAVAOPTS);
-    if (javaOpts != null && !javaOpts.isEmpty()) {
-      String logLevel = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZLOGLEVEL);
-      List<String> logProps = Lists.newArrayList();
-      TezUtils.addLog4jSystemProperties(logLevel, logProps);
-      StringBuilder sb = new StringBuilder();
-      for (String str : logProps) {
-        sb.append(str).append(" ");
+
+    String logLevel = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZLOGLEVEL);
+    List<String> logProps = Lists.newArrayList();
+    TezUtils.addLog4jSystemProperties(logLevel, logProps);
+    StringBuilder sb = new StringBuilder();
+    for (String str : logProps) {
+      sb.append(str).append(" ");
+    }
+    logLevel = sb.toString();
+
+    if (HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0) {
+      if (javaOpts != null) {
+        return javaOpts + " " + logLevel;
+      } else  {
+        return logLevel;
+      }
+    } else {
+      if (javaOpts != null && !javaOpts.isEmpty()) {
+        LOG.warn(HiveConf.ConfVars.HIVETEZJAVAOPTS + " will be ignored because "
+                 + HiveConf.ConfVars.HIVETEZCONTAINERSIZE + " is not set!");
       }
-      return javaOpts + " " + sb.toString();
+      return logLevel + " " + MRHelpers.getJavaOptsForMRMapper(conf);
     }
-    return MRHelpers.getJavaOptsForMRMapper(conf);
   }
 
   private Vertex createVertex(JobConf conf, MergeJoinWork mergeJoinWork, LocalResource appJarLr,



Mime
View raw message