tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-2445. Disable the object cleanup in local mode in LogicalIOProcessorRuntimeTask. Contributed by Jeff Zhang.
Date Wed, 13 May 2015 18:03:48 GMT
Repository: tez
Updated Branches:
  refs/heads/master ec6bbbf95 -> c048de1c5


TEZ-2445. Disable the object cleanup in local mode in LogicalIOProcessorRuntimeTask. Contributed
by Jeff Zhang.


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c048de1c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c048de1c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c048de1c

Branch: refs/heads/master
Commit: c048de1c54413076339e2f0339b1fc40f7d1f24d
Parents: ec6bbbf
Author: Siddharth Seth <sseth@apache.org>
Authored: Wed May 13 11:03:25 2015 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Wed May 13 11:03:25 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                      |  1 +
 .../runtime/LogicalIOProcessorRuntimeTask.java   | 15 +++++++++++----
 .../TestLogicalIOProcessorRuntimeTask.java       | 19 ++++++++++++++-----
 3 files changed, 26 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/c048de1c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c1e1c31..642bb65 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -22,6 +22,7 @@ INCOMPATIBLE CHANGES
     Default max limit increased. Should not affect existing users.
 
 ALL CHANGES:
+  TEZ-2445. Disable the object cleanup in local mode in LogicalIOProcessorRuntimeTask.
   TEZ-2057. tez-dag/pom.xml contains versions for dependencies.
   TEZ-1529. ATS and TezClient integration in secure kerberos enabled cluster.
   TEZ-2435. Add public key to KEYS

http://git-wip-us.apache.org/repos/asf/tez/blob/c048de1c/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
index e419a8d..24f62a0 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java
@@ -53,6 +53,7 @@ import org.apache.tez.common.RunnableWithNdc;
 import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
@@ -763,8 +764,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
     inputsMap.clear();
     outputsMap.clear();
 
-    inputSpecs.clear();
-    outputSpecs.clear();
+
 
     inputsMap.clear();
     outputsMap.clear();
@@ -772,8 +772,15 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask {
     inputContextMap.clear();
     outputContextMap.clear();
 
-    if (groupInputSpecs != null) {
-      groupInputSpecs.clear();
+    // only clean up objects in non-local mode, because local mode share the same 
+    // taskSpec in AM rather than getting it through RPC in non-local mode
+    /** Put other objects here when they are shared between AM & TezChild in local mode
**/
+    if (!tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT))
{
+      inputSpecs.clear();
+      outputSpecs.clear();
+      if (groupInputSpecs != null) {
+        groupInputSpecs.clear();
+      }
     }
     if (groupInputsMap != null) {
       groupInputsMap.clear();

http://git-wip-us.apache.org/repos/asf/tez/blob/c048de1c/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
index b337bc7..0acb7b8 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
@@ -109,7 +109,8 @@ public class TestLogicalIOProcessorRuntimeTask {
 
 
 
-
+    // local mode 
+    tezConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     LogicalIOProcessorRuntimeTask lio2 = new LogicalIOProcessorRuntimeTask(task2, 0, tezConf,
null,
         umbilical, serviceConsumerMetadata, new HashMap<String, String>(), startedInputsMap,
null,
         "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory());
@@ -158,14 +159,22 @@ public class TestLogicalIOProcessorRuntimeTask {
       assertTrue(outputContext.getUserPayload() == null);
       assertTrue(outputContext.getObjectRegistry() == null);
     }
-
-    assertEquals(0, lio.inputSpecs.size());
+    boolean localMode = lio.tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
+        TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
+    if (localMode) {
+      assertEquals(1, lio.inputSpecs.size());
+      assertEquals(1, lio.outputSpecs.size());
+      assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
+    } else {
+      assertEquals(0, lio.inputSpecs.size());
+      assertEquals(0, lio.outputSpecs.size());
+      assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
+    }
+    
     assertEquals(0, lio.inputsMap.size());
     assertEquals(0, lio.inputContextMap.size());
-    assertEquals(0, lio.outputSpecs.size());
     assertEquals(0, lio.outputsMap.size());
     assertEquals(0, lio.outputContextMap.size());
-    assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
     assertNull(lio.groupInputsMap);
     assertNull(lio.processor);
     assertNull(lio.processorContext);


Mime
View raw message