tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject git commit: TEZ-299. Remove creation of local job conf as config passed over in user payload for MR tasks. (hitesh)
Date Tue, 16 Jul 2013 19:56:04 GMT
Updated Branches:
  refs/heads/master 6ede8fef7 -> 0e2090711


TEZ-299. Remove creation of local job conf as config passed over in user payload for MR tasks.
(hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/0e209071
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/0e209071
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/0e209071

Branch: refs/heads/master
Commit: 0e2090711df41d920f61ba0647cfe453e8c40eb8
Parents: 6ede8fe
Author: Hitesh Shah <hitesh@apache.org>
Authored: Tue Jul 16 12:55:50 2013 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Tue Jul 16 12:55:50 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/common/counters/Limits.java  | 52 +++++++++++++++-----
 .../apache/hadoop/mapred/YarnTezDagChild.java   | 15 ++++--
 .../tez/mapreduce/task/MRRuntimeTask.java       |  7 ---
 3 files changed, 50 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0e209071/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
index 71b4d1b..aacce87 100644
--- a/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
+++ b/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
@@ -25,19 +25,34 @@ import org.apache.tez.common.TezJobConfig;
 @InterfaceAudience.Private
 public class Limits {
 
-  static final Configuration conf = new Configuration();
-  public static final int GROUP_NAME_MAX =
-      conf.getInt(TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY, 
-          TezJobConfig.COUNTER_GROUP_NAME_MAX_DEFAULT);
-  public static final int COUNTER_NAME_MAX =
-      conf.getInt(TezJobConfig.COUNTER_NAME_MAX_KEY, 
-          TezJobConfig.COUNTER_NAME_MAX_DEFAULT);
-  public static final int GROUPS_MAX =
-      conf.getInt(TezJobConfig.COUNTER_GROUPS_MAX_KEY, 
-          TezJobConfig.COUNTER_GROUPS_MAX_DEFAULT);
-  public static final int COUNTERS_MAX =
-      conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
-          COUNTERS_MAX_DEFAULT);
+  private static Configuration conf = null;
+  private static int GROUP_NAME_MAX;
+  private static int COUNTER_NAME_MAX;
+  private static int GROUPS_MAX;
+  private static int COUNTERS_MAX;
+  private static boolean initialized = false;
+
+  private static synchronized void ensureInitialized() {
+    if (initialized) {
+      return;
+    }
+    if (conf == null) {
+      conf = new Configuration();
+    }
+    GROUP_NAME_MAX =
+        conf.getInt(TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY,
+            TezJobConfig.COUNTER_GROUP_NAME_MAX_DEFAULT);
+    COUNTER_NAME_MAX =
+        conf.getInt(TezJobConfig.COUNTER_NAME_MAX_KEY,
+            TezJobConfig.COUNTER_NAME_MAX_DEFAULT);
+    GROUPS_MAX =
+        conf.getInt(TezJobConfig.COUNTER_GROUPS_MAX_KEY,
+            TezJobConfig.COUNTER_GROUPS_MAX_DEFAULT);
+    COUNTERS_MAX =
+        conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
+            COUNTERS_MAX_DEFAULT);
+    initialized = true;
+  }
 
   private int totalCounters;
   private LimitExceededException firstViolation;
@@ -47,14 +62,17 @@ public class Limits {
   }
 
   public static String filterCounterName(String name) {
+    ensureInitialized();
     return filterName(name, COUNTER_NAME_MAX);
   }
 
   public static String filterGroupName(String name) {
+    ensureInitialized();
     return filterName(name, GROUP_NAME_MAX);
   }
 
   public synchronized void checkCounters(int size) {
+    ensureInitialized();
     if (firstViolation != null) {
       throw new LimitExceededException(firstViolation);
     }
@@ -71,6 +89,7 @@ public class Limits {
   }
 
   public synchronized void checkGroups(int size) {
+    ensureInitialized();
     if (firstViolation != null) {
       throw new LimitExceededException(firstViolation);
     }
@@ -83,4 +102,11 @@ public class Limits {
   public synchronized LimitExceededException violation() {
     return firstViolation;
   }
+
+  public synchronized static void setConfiguration(Configuration conf) {
+    if (Limits.conf == null && conf != null) {
+      Limits.conf = conf;
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0e209071/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 4ff68af..bad68a9 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -56,6 +56,7 @@ import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.counters.Limits;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
@@ -79,12 +80,15 @@ public class YarnTezDagChild {
 
   public static void main(String[] args) throws Throwable {
     Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
-    LOG.debug("Child starting");
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Child starting");
+    }
 
     final Configuration defaultConf = new Configuration();
     // Security settings will be loaded based on core-site and core-default. Don't
     // depend on the jobConf for this.
     UserGroupInformation.setConfiguration(defaultConf);
+    Limits.setConfiguration(defaultConf);
 
     String host = args[0];
     int port = Integer.parseInt(args[1]);
@@ -101,9 +105,12 @@ public class YarnTezDagChild {
     // Security framework already loaded the tokens into current ugi
     Credentials credentials =
         UserGroupInformation.getCurrentUser().getCredentials();
-    LOG.info("Executing with tokens:");
-    for (Token<?> token: credentials.getAllTokens()) {
-      LOG.info(token);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.info("Executing with tokens:");
+      for (Token<?> token: credentials.getAllTokens()) {
+        LOG.info(token);
+      }
     }
 
     // Create TaskUmbilicalProtocol as actual task owner.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0e209071/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 1a3a5d4..3e6a95a 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -209,13 +209,6 @@ public class MRRuntimeTask extends RuntimeTask {
     // Set up the DistributedCache related configs
     setupDistributedCacheConfig(job);
 
-    // Overwrite the localized task jobconf which is linked to in the current
-    // work-dir.
-    Path localTaskFile = new Path(
-        job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
-        MRJobConfig.JOB_CONF_FILE);
-    writeLocalJobFile(localTaskFile, job);
-
     task.setConf(job);
   }
 


Mime
View raw message