tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: Addendum TEZ-510. Test MRRJobs failing with NPE (bikas)
Date Thu, 26 Sep 2013 21:23:38 GMT
Updated Branches:
  refs/heads/master 968e31f96 -> 2debbca59


Addendum TEZ-510. Test MRRJobs failing with NPE (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/2debbca5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/2debbca5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/2debbca5

Branch: refs/heads/master
Commit: 2debbca5921daa8fd4a9080e8390c0c086914d62
Parents: 968e31f
Author: Bikas Saha <bikas@apache.org>
Authored: Thu Sep 26 14:21:42 2013 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Thu Sep 26 14:21:42 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/dag/app/dag/impl/VertexImpl.java    | 11 +++++------
 .../tez/processor/FilterByWordInputProcessor.java      | 13 +++++++------
 .../java/org/apache/tez/mapreduce/input/MRInput.java   |  5 ++++-
 3 files changed, 16 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2debbca5/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 6752328..99dbb87 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -250,12 +250,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
               EnumSet.of(VertexState.TERMINATING, VertexState.KILLED, VertexState.FAILED),
               VertexEventType.V_TASK_COMPLETED,
               new TaskCompletedTransition())
-          .addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
-              VertexEventType.V_TASK_ATTEMPT_COMPLETED,
-              TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) // TODO shouldnt be done for KILL_WAIT
vertex
-          .addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
-              VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED,
-              SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
           .addTransition(
               VertexState.TERMINATING,
               VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR,
@@ -263,6 +257,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
           // Ignore-able events
           .addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
               EnumSet.of(VertexEventType.V_TERMINATE,
+                  VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED,
+                  VertexEventType.V_TASK_ATTEMPT_COMPLETED,
                   VertexEventType.V_TASK_RESCHEDULED))
 
           // Transitions from SUCCEEDED state
@@ -301,6 +297,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
           // Ignore-able events
           .addTransition(VertexState.KILLED, VertexState.KILLED,
               EnumSet.of(VertexEventType.V_TERMINATE,
+                  VertexEventType.V_SOURCE_VERTEX_STARTED,
                   VertexEventType.V_START,
                   VertexEventType.V_TASK_RESCHEDULED,
                   VertexEventType.V_TASK_ATTEMPT_COMPLETED,
@@ -311,6 +308,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
               VertexState.ERROR,
               VertexState.ERROR,
               EnumSet.of(VertexEventType.V_INIT,
+                  VertexEventType.V_SOURCE_VERTEX_STARTED,
+                  VertexEventType.V_START,
                   VertexEventType.V_TERMINATE,
                   VertexEventType.V_TASK_COMPLETED,
                   VertexEventType.V_TASK_ATTEMPT_COMPLETED,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2debbca5/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
index e8e315a..6c97430 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
@@ -98,13 +98,14 @@ public class FilterByWordInputProcessor implements LogicalIOProcessor
{
     OnFileUnorderedKVOutput kvOutput = (OnFileUnorderedKVOutput) lo;
 
     Configuration updatedConf = mrInput.getConfigUpdates();
-    String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
-    LOG.info("Processing file: " + fileName);
     Text srcFile = new Text();
-    if (fileName == null) {
-      srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
-    } else {
-      srcFile.set(fileName);
+    srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
+    if (updatedConf != null) {
+      String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
+      if (fileName != null) {
+        LOG.info("Processing file: " + fileName);
+        srcFile.set(fileName);
+      }
     }
 
     KVReader kvReader = mrInput.getReader();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2debbca5/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index b9f2242..ed675a4 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -209,7 +209,10 @@ public class MRInput implements LogicalInput {
    * @return the additional fields set by {@link MRInput}
    */
   public Configuration getConfigUpdates() {
-    return new Configuration(incrementalConf);
+    if (incrementalConf != null) {
+      return new Configuration(incrementalConf);
+    }
+    return null;
   }
 
   public float getProgress() throws IOException, InterruptedException {


Mime
View raw message