tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject [15/50] [abbrv] git commit: TEZ-9. Add support for client api and records. cleaned up packages (bikas)
Date Tue, 04 Jun 2013 05:33:17 GMT
TEZ-9. Add support for client api and records. cleaned up packages (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/de5ee038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/de5ee038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/de5ee038

Branch: refs/heads/master
Commit: de5ee0382085e91e5644d881182309bc6858e9ed
Parents: cee79f6
Author: Bikas Saha <bikas@apache.org>
Authored: Fri May 17 15:59:37 2013 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Fri May 17 15:59:37 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/dag/records/TezDAGID.java  |   14 ++
 tez-dag-api/pom.xml                                |    3 +-
 .../src/main/java/org/apache/tez/dag/api/DAG.java  |   16 +-
 .../org/apache/tez/dag/api/DagTypeConverters.java  |   18 +-
 .../org/apache/tez/dag/api/client/DAGClient.java   |   46 ++++
 .../org/apache/tez/dag/api/client/DAGStatus.java   |   95 ++++++++
 .../org/apache/tez/dag/api/client/Progress.java    |   51 +++++
 .../apache/tez/dag/api/client/VertexStatus.java    |   75 +++++++
 .../client/rpc/DAGClientAMProtocolBlockingPB.java  |   30 +++
 .../tez/dag/api/client/rpc/DAGClientRPCImpl.java   |  103 +++++++++
 tez-dag-api/src/main/proto/DAGApiRecords.proto     |  167 +++++++++++++++
 .../src/main/proto/DAGClientAMProtocol.proto       |   56 +++++
 tez-dag-api/src/main/proto/DAGPlan.proto           |  119 ----------
 .../java/org/apache/tez/dag/api/TestDAGPlan.java   |   10 +-
 .../hadoop/mapred/MRVertexOutputCommitter.java     |    8 +-
 .../tez/dag/api/client/DAGStatusBuilder.java       |   86 ++++++++
 .../apache/tez/dag/api/client/ProgressBuilder.java |   58 +++++
 .../tez/dag/api/client/VertexStatusBuilder.java    |   78 +++++++
 .../tez/dag/api/client/impl/TezBuilderUtils.java   |   54 -----
 .../apache/tez/dag/api/committer/DAGStatus.java    |   39 ++++
 .../api/committer/NullVertexOutputCommitter.java   |   47 ++++
 .../tez/dag/api/committer/VertexContext.java       |   37 ++++
 .../dag/api/committer/VertexOutputCommitter.java   |   69 ++++++
 .../apache/tez/dag/api/committer/VertexStatus.java |   51 +++++
 .../org/apache/tez/dag/api/impl/DAGStatus.java     |   39 ----
 .../dag/api/impl/NullVertexOutputCommitter.java    |   47 ----
 .../org/apache/tez/dag/api/impl/VertexContext.java |   37 ----
 .../tez/dag/api/impl/VertexOutputCommitter.java    |   69 ------
 .../org/apache/tez/dag/api/impl/VertexStatus.java  |   51 -----
 .../org/apache/tez/dag/api/oldrecords/AMInfo.java  |   38 ++++
 .../tez/dag/api/oldrecords/TaskAttemptReport.java  |   66 ++++++
 .../tez/dag/api/oldrecords/TaskAttemptState.java   |   29 +++
 .../apache/tez/dag/api/oldrecords/TaskReport.java  |   63 ++++++
 .../apache/tez/dag/api/oldrecords/TaskState.java   |   23 ++
 .../org/apache/tez/dag/api/records/AMInfo.java     |   38 ----
 .../tez/dag/api/records/TaskAttemptReport.java     |   66 ------
 .../tez/dag/api/records/TaskAttemptState.java      |   29 ---
 .../org/apache/tez/dag/api/records/TaskReport.java |   63 ------
 .../org/apache/tez/dag/api/records/TaskState.java  |   23 --
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |   41 ++++-
 .../org/apache/tez/dag/app/MRRExampleHelper.java   |    2 +-
 .../main/java/org/apache/tez/dag/app/dag/DAG.java  |    7 +-
 .../java/org/apache/tez/dag/app/dag/DAGReport.java |    2 +-
 .../main/java/org/apache/tez/dag/app/dag/Task.java |    4 +-
 .../org/apache/tez/dag/app/dag/TaskAttempt.java    |    4 +-
 .../java/org/apache/tez/dag/app/dag/Vertex.java    |    6 +-
 .../dag/event/TaskAttemptEventStatusUpdate.java    |    2 +-
 .../app/dag/event/VertexEventTaskCompleted.java    |    2 +-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |   58 +++++-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |    6 +-
 .../org/apache/tez/dag/app/dag/impl/TaskImpl.java  |    6 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   51 ++++-
 .../tez/dag/app/rm/AMSchedulerEventTAEnded.java    |    2 +-
 .../tez/dag/app/rm/TaskSchedulerEventHandler.java  |    2 +-
 .../tez/dag/app/speculate/DefaultSpeculator.java   |    4 +-
 .../app/speculate/LegacyTaskRuntimeEstimator.java  |    2 +-
 .../tez/dag/app/speculate/StartEndTimesBase.java   |    2 +-
 .../tez/dag/history/events/DAGFinishedEvent.java   |    2 +-
 .../history/events/TaskAttemptFinishedEvent.java   |    2 +-
 .../tez/dag/history/events/TaskFinishedEvent.java  |    2 +-
 .../dag/history/events/VertexFinishedEvent.java    |    2 +-
 .../org/apache/tez/dag/utils/TezBuilderUtils.java  |   54 +++++
 .../tez/dag/app/dag/impl/TestTaskAttempt.java      |    2 +-
 .../apache/tez/dag/app/dag/impl/TestTaskImpl.java  |    4 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |   26 ++--
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |    2 +-
 66 files changed, 1587 insertions(+), 723 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index e977dc0..733a908 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -100,6 +100,20 @@ public class TezDAGID extends TezID {
   public String toString() {
     return appendTo(new StringBuilder(DAG)).toString();
   }
+  
+  public static TezDAGID fromString(String dagId) {
+    try {
+      String[] split = dagId.split("_");
+      String rmId = split[1];
+      int appId = Integer.parseInt(split[2]);
+      int id;
+      id = idFormat.parse(split[3]).intValue();
+      return new TezDAGID(rmId, appId, id);
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+    return null;
+  }
 
   @Override
   public void readFields(DataInput in) throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 2b6aebb..74b1284 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -64,7 +64,8 @@
               <source>
                 <directory>${basedir}/src/main/proto</directory>
                 <includes>
-                  <include>DAGPlan.proto</include>
+                  <include>DAGApiRecords.proto</include>
+                  <include>DAGClientAMProtocol.proto</include>
                 </includes>
               </source>
               <output>${project.build.directory}/generated-sources/java</output>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 300ef31..7fa764a 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -28,14 +28,14 @@ import java.util.Stack;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 
 public class DAG { // FIXME rename to Topology

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index c5fd0a2..ba42e23 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -31,15 +31,15 @@ import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceVisibility;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResourceVisibility;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty.SourceType;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
new file mode 100644
index 0000000..477b017
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.TezException;
+
+/*
+ * Inteface class for monitoring the <code>DAG</code> running in a Tez DAG
+ * Application Master.
+ */
+public interface DAGClient {
+  
+  /**
+   * Return the identifiers for all DAG's
+   */
+  List<String> getAllDAGs() throws TezException;
+  
+  /**
+   * Get the status of a DAG
+   */
+  DAGStatus getDAGStatus(String dagId) throws TezException;
+  
+  /**
+   * Get the status of a Vertex of a DAG 
+   */
+  VertexStatus getVertexStatus(String dagId, 
+                               String vertexName) throws TezException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
new file mode 100644
index 0000000..597789b
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProtoOrBuilder;
+import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
+import org.apache.tez.dag.api.TezException;
+
+public class DAGStatus {
+
+  public enum State {
+    SUBMITTED,
+    INITING,
+    RUNNING,
+    SUCCEEDED,
+    KILLED,
+    FAILED,
+    ERROR
+  };
+  
+  DAGStatusProtoOrBuilder proxy = null;
+  Progress progress = null;
+  Map<String, Progress> vertexProgress = null;
+  
+  public DAGStatus(DAGStatusProtoOrBuilder proxy) {
+    this.proxy = proxy;
+  }
+  
+  public State getState() {
+    switch(proxy.getState()) {
+    case DAG_SUBMITTED:
+      return DAGStatus.State.SUBMITTED;
+    case DAG_INITING:
+      return DAGStatus.State.INITING;
+    case DAG_RUNNING:
+      return DAGStatus.State.RUNNING;
+    case DAG_SUCCEEDED:
+      return DAGStatus.State.SUCCEEDED;
+    case DAG_FAILED:
+      return DAGStatus.State.FAILED;
+    case DAG_KILLED:
+      return DAGStatus.State.KILLED;
+    case DAG_ERROR:
+      return DAGStatus.State.ERROR;
+    default:
+      throw new TezException("Unsupported value for DAGStatus.State : " + 
+                              proxy.getState());
+    }
+  }
+
+  public List<String> getDiagnostics() {
+    return proxy.getDiagnosticsList();
+  }
+
+  public Progress getDAGProgress() {
+    if(progress == null && proxy.hasDAGProgress()) {
+      progress = new Progress(proxy.getDAGProgress());
+    }
+    return progress;
+  }
+
+  public Map<String, Progress> getVertexProgress() {
+    if(vertexProgress == null) {
+      if(proxy.getVertexProgressList() != null) {
+        List<StringProgressPairProto> kvList = proxy.getVertexProgressList();
+        vertexProgress = new HashMap<String, Progress>(kvList.size());        
+        for(StringProgressPairProto kv : kvList){
+          vertexProgress.put(kv.getKey(), new Progress(kv.getProgress()));
+        }
+      }
+    }
+    return vertexProgress;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
new file mode 100644
index 0000000..7158e30
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProtoOrBuilder;
+
+public class Progress {
+  
+  ProgressProtoOrBuilder proxy = null;
+  
+  Progress(ProgressProtoOrBuilder proxy) {
+    this.proxy = proxy;
+  }
+  
+  public int getTotalTaskCount() {
+    return proxy.getTotalTaskCount();
+  }
+
+  public int getSucceededTaskCount() {
+    return proxy.getSucceededTaskCount();
+  }
+
+  public int getRunningTaskCount() {
+    return proxy.getRunningTaskCount();
+  }
+
+  public int getFailedTaskCount() {
+    return proxy.getFailedTaskCount();
+  }
+
+  public int getKilledTaskCount() {
+    return proxy.getKilledTaskCount();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
new file mode 100644
index 0000000..33b4de3
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
@@ -0,0 +1,75 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProtoOrBuilder;
+import org.apache.tez.dag.api.TezException;
+
+public class VertexStatus {
+  
+  public enum State {
+    INITED,
+    RUNNING,
+    SUCCEEDED,
+    KILLED,
+    FAILED,
+    ERROR
+  };
+  
+  VertexStatusProtoOrBuilder proxy = null;
+  Progress progress = null;
+  
+  public VertexStatus(VertexStatusProtoOrBuilder proxy) {
+    this.proxy = proxy;
+  }
+
+  public State getState() {
+    switch(proxy.getState()) {
+    case VERTEX_INITED:
+      return VertexStatus.State.INITED;
+    case VERTEX_RUNNING:
+      return VertexStatus.State.RUNNING;
+    case VERTEX_SUCCEEDED:
+      return VertexStatus.State.SUCCEEDED;
+    case VERTEX_FAILED:
+      return VertexStatus.State.FAILED;
+    case VERTEX_KILLED:
+      return VertexStatus.State.KILLED;
+    case VERTEX_ERROR:
+      return VertexStatus.State.ERROR;
+    default:
+      throw new TezException("Unsupported value for VertexStatus.State : " + 
+                              proxy.getState());
+    }    
+  }
+
+  public List<String> getDiagnostics() {
+    return proxy.getDiagnosticsList();
+  }
+
+  public Progress getProgress() {
+    if(progress == null && proxy.hasProgress()) {
+      progress = new Progress(proxy.getProgress());
+    }
+    return progress;    
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
new file mode 100644
index 0000000..a1ee18f
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.rpc;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol;
+
+@ProtocolInfo(
+    protocolName = "org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolPB",
+    protocolVersion = 1)
+public interface DAGClientAMProtocolBlockingPB 
+                              extends DAGClientAMProtocol.BlockingInterface {
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
new file mode 100644
index 0000000..9238f63
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.rpc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
+
+import com.google.protobuf.ServiceException;
+
+public class DAGClientRPCImpl implements DAGClient, Closeable {
+
+  private DAGClientAMProtocolBlockingPB proxy = null;
+  
+  public DAGClientRPCImpl(long clientVersion, 
+                          InetSocketAddress addr,
+                          TezConfiguration conf) throws IOException {
+    RPC.setProtocolEngine(conf, 
+                          DAGClientAMProtocolBlockingPB.class, 
+                          ProtobufRpcEngine.class);
+    proxy =
+        (DAGClientAMProtocolBlockingPB) 
+        RPC.getProxy(DAGClientAMProtocolBlockingPB.class, 
+                     clientVersion,
+                     addr, 
+                     conf);
+  }
+  
+  @Override
+  public List<String> getAllDAGs() throws TezException {
+    GetAllDAGsRequestProto requestProto = 
+        GetAllDAGsRequestProto.newBuilder().build();
+    try {
+      return proxy.getAllDAGs(null, requestProto).getDagIdList();
+    } catch (ServiceException e) {
+      throw new TezException(e);
+    }
+  }
+
+  @Override
+  public DAGStatus getDAGStatus(String dagId) throws TezException {
+    GetDAGStatusRequestProto requestProto = 
+        GetDAGStatusRequestProto.newBuilder().setDagId(dagId).build();
+    
+    try {
+      return new DAGStatus(
+                 proxy.getDAGStatus(null, requestProto).getDagStatus());
+    } catch (ServiceException e) {
+      throw new TezException(e);
+    }
+  }
+
+  @Override
+  public VertexStatus getVertexStatus(String dagId, String vertexName)
+      throws TezException {
+    GetVertexStatusRequestProto requestProto = 
+        GetVertexStatusRequestProto.newBuilder().
+                        setDagId(dagId).setVertexName(vertexName).build();
+    
+    try {
+      return new VertexStatus(
+                 proxy.getVertexStatus(null, requestProto).getVertexStatus());
+    } catch (ServiceException e) {
+      throw new TezException(e);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (this.proxy != null) {
+      RPC.stopProxy(this.proxy);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGApiRecords.proto b/tez-dag-api/src/main/proto/DAGApiRecords.proto
new file mode 100644
index 0000000..242715b
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGApiRecords.proto
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api.records";
+option java_outer_classname = "DAGProtos";
+option java_generate_equals_and_hash = true;
+
+// DAG plan messages
+
+// Many of these types have a dual in the Tez-api.  To reduce confusion, these types have prefix or suffix 
+// of "Plan" to indicate they are to be used in the dag-plan.
+// The big types use a suffix:  JobPlan, VertexPlan, EdgePlan 
+//   --> these get more direct use in the runtime and the naming is natural.
+// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
+//   --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable. 
+
+enum PlanVertexType {
+  INPUT = 0;
+  NORMAL = 1;
+  OUTPUT = 2;
+}
+
+enum PlanEdgeConnectionPattern {
+  ONE_TO_ONE = 0;
+  ONE_TO_ALL = 1;
+  BIPARTITE = 2;
+}
+
+enum PlanEdgeSourceType {
+  STABLE = 0;
+  STABLE_PERSISTED = 1;
+  STREAMING = 2;
+}
+
+message PlanKeyValuePair {
+  required string key = 1;
+  required string value = 2;
+}
+
+enum PlanLocalResourceType {
+  FILE = 0;
+  ARCHIVE = 1;
+  PATTERN = 2;
+}
+
+enum PlanLocalResourceVisibility {
+  PUBLIC = 0;
+  PRIVATE = 1;
+  APPLICATION = 2;
+}
+
+message PlanLocalResource {
+  required string name = 1;
+  required string uri = 2;
+  required int64 size = 3;
+  required int64 timeStamp = 4;
+  required PlanLocalResourceType type = 5;
+  required PlanLocalResourceVisibility visibility = 6;
+  optional string pattern = 7; // only used if type=PATTERN
+}
+
+// Each taskLocationHint represents a single split in in the input.
+// It is the list of [{rack,machines}] that host a replica of each particular split.
+// For now it is represented as pair-of-arrays rather than array-of-pairs.
+message PlanTaskLocationHint {
+  repeated string rack = 1;
+  repeated string host = 2;
+}
+
+message PlanTaskConfiguration {
+  required int32 numTasks = 1;
+  required int32 memoryMb = 2;
+  required int32 virtualCores = 3;
+  required string javaOpts = 4;
+  required string taskModule = 5;
+  repeated PlanLocalResource localResource = 6;
+  repeated PlanKeyValuePair environmentSetting = 8;  
+}
+
+message VertexPlan {
+  required string name = 1;
+  required PlanVertexType type = 2;
+  optional string processorName = 3;
+  required PlanTaskConfiguration taskConfig = 4;
+  repeated PlanTaskLocationHint taskLocationHint = 7;
+  repeated string inEdgeId = 8;
+  repeated string outEdgeId = 9;
+}
+
+message EdgePlan {
+  required string id = 1;
+  required string inputVertexName = 2;
+  required string outputVertexName = 3;
+  required PlanEdgeConnectionPattern connectionPattern = 4;
+  required PlanEdgeSourceType sourceType = 5;
+  required string inputClass = 6;
+  required string outputClass = 7;
+}
+
+message DAGPlan {
+  required string name = 1;
+  repeated VertexPlan vertex = 2;
+  repeated EdgePlan edge = 3;
+  repeated PlanKeyValuePair jobSetting = 4;
+}
+
+
+// DAG monitoring messages
+message ProgressProto {
+  optional int32 totalTaskCount = 1;
+  optional int32 succeededTaskCount = 2;
+  optional int32 runningTaskCount = 3;
+  optional int32 failedTaskCount = 4;
+  optional int32 killedTaskCount = 5;
+}
+
+enum VertexStatusStateProto {
+  VERTEX_INITED = 0;
+  VERTEX_RUNNING = 1;
+  VERTEX_SUCCEEDED = 2;
+  VERTEX_KILLED = 3;
+  VERTEX_FAILED = 4;
+  VERTEX_ERROR = 5;
+}
+
+message VertexStatusProto {
+  optional VertexStatusStateProto state = 1;
+  repeated string diagnostics = 2;
+  optional ProgressProto progress = 3;
+}
+
+enum DAGStatusStateProto {
+  DAG_SUBMITTED = 0;
+  DAG_INITING = 1;
+  DAG_RUNNING = 2;
+  DAG_SUCCEEDED = 3;
+  DAG_KILLED = 4;
+  DAG_FAILED = 5;
+  DAG_ERROR = 6;
+}
+
+message StringProgressPairProto {
+  required string key = 1;
+  required ProgressProto progress = 2;
+}
+
+message DAGStatusProto {
+  optional DAGStatusStateProto state = 1;
+  repeated string diagnostics = 2;
+  optional ProgressProto DAGProgress = 3;
+  repeated StringProgressPairProto vertexProgress = 4;  
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto b/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
new file mode 100644
index 0000000..18247a1
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api.client.rpc";
+option java_outer_classname = "DAGClientAMProtocolRPC";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+//import "DAGClientAMProtocolRecords.proto";
+
+import "DAGApiRecords.proto";
+
+message GetAllDAGsRequestProto {
+}
+
+message GetAllDAGsResponseProto {
+  repeated string dagId = 1;
+}
+
+message GetDAGStatusRequestProto {
+  optional string dagId = 1;
+}
+
+message GetDAGStatusResponseProto {
+  optional DAGStatusProto dagStatus = 1; 
+}
+
+message GetVertexStatusRequestProto {
+  optional string dagId = 1;
+  optional string vertexName = 2;
+}
+
+message GetVertexStatusResponseProto {
+  optional VertexStatusProto vertexStatus = 1;
+}
+
+service DAGClientAMProtocol {
+  rpc getAllDAGs (GetAllDAGsRequestProto) returns (GetAllDAGsResponseProto);
+  rpc getDAGStatus (GetDAGStatusRequestProto) returns (GetDAGStatusResponseProto);
+  rpc getVertexStatus (GetVertexStatusRequestProto) returns (GetVertexStatusResponseProto);
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
deleted file mode 100644
index 30fc54e..0000000
--- a/tez-dag-api/src/main/proto/DAGPlan.proto
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.tez.dag.api";
-option java_outer_classname = "DAGProtos";
-option java_generate_equals_and_hash = true;
-package org.apache.tez.dag.api;
-
-// Many of these types have a dual in the Tez-api.  To reduce confusion, these types have prefix or suffix 
-// of "Plan" to indicate they are to be used in the dag-plan.
-// The big types use a suffix:  JobPlan, VertexPlan, EdgePlan 
-//   --> these get more direct use in the runtime and the naming is natural.
-// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
-//   --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable. 
-
-enum PlanVertexType {
-  INPUT = 0;
-  NORMAL = 1;
-  OUTPUT = 2;
-}
-
-enum PlanEdgeConnectionPattern {
-  ONE_TO_ONE = 0;
-  ONE_TO_ALL = 1;
-  BIPARTITE = 2;
-}
-
-enum PlanEdgeSourceType {
-  STABLE = 0;
-  STABLE_PERSISTED = 1;
-  STREAMING = 2;
-}
-
-message PlanKeyValuePair {
-  required string key = 1;
-  required string value = 2;
-}
-
-enum PlanLocalResourceType {
-  FILE = 0;
-  ARCHIVE = 1;
-  PATTERN = 2;
-}
-
-enum PlanLocalResourceVisibility {
-  PUBLIC = 0;
-  PRIVATE = 1;
-  APPLICATION = 2;
-}
-
-message PlanLocalResource {
-  required string name = 1;
-  required string uri = 2;
-  required int64 size = 3;
-  required int64 timeStamp = 4;
-  required PlanLocalResourceType type = 5;
-  required PlanLocalResourceVisibility visibility = 6;
-  optional string pattern = 7; // only used if type=PATTERN
-}
-
-// Each taskLocationHint represents a single split in in the input.
-// It is the list of [{rack,machines}] that host a replica of each particular split.
-// For now it is represented as pair-of-arrays rather than array-of-pairs.
-message PlanTaskLocationHint {
-  repeated string rack = 1;
-  repeated string host = 2;
-}
-
-message PlanTaskConfiguration {
-  required int32 numTasks = 1;
-  required int32 memoryMb = 2;
-  required int32 virtualCores = 3;
-  required string javaOpts = 4;
-  required string taskModule = 5;
-  repeated PlanLocalResource localResource = 6;
-  repeated PlanKeyValuePair environmentSetting = 8;  
-}
-
-message VertexPlan {
-  required string name = 1;
-  required PlanVertexType type = 2;
-  optional string processorName = 3;
-  required PlanTaskConfiguration taskConfig = 4;
-  repeated PlanTaskLocationHint taskLocationHint = 7;
-  repeated string inEdgeId = 8;
-  repeated string outEdgeId = 9;
-}
-
-message EdgePlan {
-  required string id = 1;
-  required string inputVertexName = 2;
-  required string outputVertexName = 3;
-  required PlanEdgeConnectionPattern connectionPattern = 4;
-  required PlanEdgeSourceType sourceType = 5;
-  required string inputClass = 6;
-  required string outputClass = 7;
-}
-
-message DAGPlan {
-  required string name = 1;
-  repeated VertexPlan vertex = 3;
-  repeated EdgePlan edge = 4;
-  repeated PlanKeyValuePair jobSetting = 5;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index a041f41..6dca610 100644
--- a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -23,11 +23,11 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index a3b72c7..7c886d6 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.YarnException;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 public class MRVertexOutputCommitter extends VertexOutputCommitter {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
new file mode 100644
index 0000000..553347f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
@@ -0,0 +1,86 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusStateProto;
+import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto;
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto.Builder;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.app.dag.DAGState;
+
+public class DAGStatusBuilder extends DAGStatus {
+
+  public DAGStatusBuilder() {
+    super(DAGStatusProto.newBuilder());
+  }
+  
+  public void setState(DAGState state) {
+    getBuilder().setState(getProtoState(state));
+  }
+  
+  public void setDiagnostics(List<String> diagnostics) {
+    Builder builder = getBuilder();
+    builder.clearDiagnostics();
+    builder.addAllDiagnostics(diagnostics);
+  }
+  
+  public void setDAGProgress(ProgressBuilder progress) {
+    getBuilder().setDAGProgress(progress.getProto());
+  }
+  
+  public void addVertexProgress(String name, ProgressBuilder progress) {
+    StringProgressPairProto.Builder builder = StringProgressPairProto.newBuilder();
+    builder.setKey(name);
+    builder.setProgress(progress.getProto());
+    getBuilder().addVertexProgress(builder.build());
+  }
+  
+  public DAGStatusProto getProto() {
+    return getBuilder().build();
+  }
+  
+  private DAGStatusStateProto getProtoState(DAGState state) {
+    switch(state) {
+    case NEW:
+    case INITED:
+      return DAGStatusStateProto.DAG_INITING;
+    case RUNNING:
+      return DAGStatusStateProto.DAG_RUNNING;
+    case SUCCEEDED:
+      return DAGStatusStateProto.DAG_SUCCEEDED;
+    case FAILED:
+      return DAGStatusStateProto.DAG_FAILED;
+    case KILLED:
+    case KILL_WAIT:
+      return DAGStatusStateProto.DAG_KILLED;
+    case ERROR:
+      return DAGStatusStateProto.DAG_ERROR;
+    default:
+      throw new TezException("Unsupported value for DAGState : " + state);
+    }
+  }
+  
+  private DAGStatusProto.Builder getBuilder() {
+    return (Builder) this.proxy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
new file mode 100644
index 0000000..6cedb3f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProto;
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProto.Builder;
+import org.apache.tez.dag.api.client.Progress;
+
+public class ProgressBuilder extends Progress {
+
+  public ProgressBuilder() {
+    super(ProgressProto.newBuilder());
+  }
+  
+  public ProgressProto getProto() {
+    return getBuilder().build();
+  }
+  
+  public void setTotalTaskCount(int count) {
+    getBuilder().setTotalTaskCount(count);
+  }
+  
+  public void setSucceededTaskCount(int count) {
+    getBuilder().setSucceededTaskCount(count);
+  }
+  
+  public void setRunningTaskCount(int count) {
+    getBuilder().setRunningTaskCount(count);
+  }
+  
+  public void setFailedTaskCount(int count) {
+    getBuilder().setFailedTaskCount(count);
+  }
+  
+  public void setKilledTaskCount(int count) {
+    getBuilder().setKilledTaskCount(count);
+  }
+  
+  private ProgressProto.Builder getBuilder() {
+    return (Builder) this.proxy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
new file mode 100644
index 0000000..91e3e83
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
@@ -0,0 +1,78 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto;
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto.Builder;
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusStateProto;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.dag.VertexState;
+
+public class VertexStatusBuilder extends VertexStatus {
+
+  public VertexStatusBuilder() {
+    super(VertexStatusProto.newBuilder());
+  }
+  
+  public void setState(VertexState state) {
+    getBuilder().setState(getProtoState(state));
+  }
+  
+  public void setDiagnostics(List<String> diagnostics) {
+    Builder builder = getBuilder();
+    builder.clearDiagnostics();
+    builder.addAllDiagnostics(diagnostics);
+  }
+  
+  public void setProgress(ProgressBuilder progress) {
+    getBuilder().setProgress(progress.getProto());
+  }
+  
+  public VertexStatusProto getProto() {
+    return getBuilder().build();
+  }
+  
+  private VertexStatusStateProto getProtoState(VertexState state) {
+    switch(state) {
+    case NEW:
+    case INITED:
+      return VertexStatusStateProto.VERTEX_INITED;
+    case RUNNING:
+      return VertexStatusStateProto.VERTEX_RUNNING;
+    case SUCCEEDED:
+      return VertexStatusStateProto.VERTEX_SUCCEEDED;
+    case FAILED:
+      return VertexStatusStateProto.VERTEX_FAILED;
+    case KILLED:
+    case KILL_WAIT:
+      return VertexStatusStateProto.VERTEX_KILLED;
+    case ERROR:
+      return VertexStatusStateProto.VERTEX_ERROR;
+    default:
+      throw new TezException("Unsupported value for VertexState : " + state);
+    }
+  }
+  
+  private VertexStatusProto.Builder getBuilder() {
+    return (Builder) this.proxy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
deleted file mode 100644
index 5f36a58..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.client.impl;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.api.records.AMInfo;
-import org.apache.tez.dag.app.dag.DAGReport;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public class TezBuilderUtils {
-
-  public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) {
-    return new TezVertexID(dagId, vertexId);
-  }
-
-  public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) {
-    return new TezTaskAttemptID(taskId, id);
-  }
-  
-  public static DAGReport newDAGReport() {
-    return null;
-  }
-
-  public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, 
-      long startTime, ContainerId containerID, String nmHost, 
-      int nmPort, int nmHttpPort) {
-    return null;
-  }
-
-  public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) {
-    return new TezTaskID(newVertexID(dagId, vertexId), taskId);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
new file mode 100644
index 0000000..587e18f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.committer;
+
+public class DAGStatus {
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4),
+    KILLED(5);
+    
+    int value;
+    
+    State(int value) {
+      this.value = value;
+    }
+    
+    public int getValue() {
+      return value; 
+    }
+    
+  };
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
new file mode 100644
index 0000000..c284ab5
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import java.io.IOException;
+
+import org.apache.tez.dag.api.committer.VertexStatus.State;
+
+public class NullVertexOutputCommitter extends VertexOutputCommitter {
+
+  @Override
+  public void init(VertexContext context) throws IOException {
+    // Nothing to do
+  }
+
+  @Override
+  public void setupVertex() throws IOException {
+    // Nothing to do
+  }
+
+  @Override
+  public void commitVertex() throws IOException {
+    // Nothing to do
+  }
+
+  @Override
+  public void abortVertex(State finalState) throws IOException {
+    // Nothing to do
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
new file mode 100644
index 0000000..14880d2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
+
+public interface VertexContext {
+
+  public Configuration getConf();
+
+  public TezDAGID getDAGId();
+  
+  // TODO Get rid of this as part of VertexContext cleanup.
+  public ApplicationAttemptId getApplicationAttemptId();
+
+  public TezVertexID getVertexId();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
new file mode 100644
index 0000000..b814d0e
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Public
+@Unstable
+public abstract class VertexOutputCommitter {
+
+  /**
+   * Setup up the vertex output committer.
+   *
+   * @param context Context of the vertex whose output is being written.
+   * @throws IOException
+   */
+  public abstract void init(VertexContext context) throws IOException;
+
+  /**
+   * For the framework to setup the vertex output during initialization. This is
+   * called from the application master process for the vertex. This will be
+   * called multiple times, once per dag attempt for each vertex.
+   *
+   * @throws IOException if setup fails
+   */
+  public abstract void setupVertex() throws IOException;
+
+  /**
+   * For committing vertex's output after successful vertex completion.
+   * Note that this is invoked for vertices with a successful final state.
+   * This is called from the application master process for the entire vertex.
+   * This is guaranteed to only be called once.
+   * If it throws an exception the entire vertex will fail.
+   *
+   * @throws IOException
+   */
+  public abstract void commitVertex() throws IOException;
+
+  /**
+   * For aborting an unsuccessful vertex's output. Note that this is invoked for
+   * vertices with a final failed state. This is called from the application
+   * master process for the entire vertex. This may be called multiple times.
+   *
+   * @param state final runstate of the vertex
+   * @throws IOException
+   */
+  public abstract void abortVertex(VertexStatus.State finalState)
+      throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
new file mode 100644
index 0000000..a78027a
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.committer;
+
+public class VertexStatus {
+
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4), // TODO change to INITING
+    KILLED(5);
+
+    int value;
+
+    State(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return value;
+    }
+
+  };
+
+  private State runState;
+
+  public synchronized State getState() {
+    return runState;
+  }
+
+  public synchronized void setState(State state) {
+    this.runState = state;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
deleted file mode 100644
index 6ede497..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.impl;
-
-public class DAGStatus {
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-    
-    int value;
-    
-    State(int value) {
-      this.value = value;
-    }
-    
-    public int getValue() {
-      return value; 
-    }
-    
-  };
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
deleted file mode 100644
index 102bdec..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import java.io.IOException;
-
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-
-public class NullVertexOutputCommitter extends VertexOutputCommitter {
-
-  @Override
-  public void init(VertexContext context) throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void setupVertex() throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void commitVertex() throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void abortVertex(State finalState) throws IOException {
-    // Nothing to do
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
deleted file mode 100644
index 181aea5..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public interface VertexContext {
-
-  public Configuration getConf();
-
-  public TezDAGID getDAGId();
-  
-  // TODO Get rid of this as part of VertexContext cleanup.
-  public ApplicationAttemptId getApplicationAttemptId();
-
-  public TezVertexID getVertexId();
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
deleted file mode 100644
index fe3e2d5..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-@Public
-@Unstable
-public abstract class VertexOutputCommitter {
-
-  /**
-   * Setup up the vertex output committer.
-   *
-   * @param context Context of the vertex whose output is being written.
-   * @throws IOException
-   */
-  public abstract void init(VertexContext context) throws IOException;
-
-  /**
-   * For the framework to setup the vertex output during initialization. This is
-   * called from the application master process for the vertex. This will be
-   * called multiple times, once per dag attempt for each vertex.
-   *
-   * @throws IOException if setup fails
-   */
-  public abstract void setupVertex() throws IOException;
-
-  /**
-   * For committing vertex's output after successful vertex completion.
-   * Note that this is invoked for vertices with a successful final state.
-   * This is called from the application master process for the entire vertex.
-   * This is guaranteed to only be called once.
-   * If it throws an exception the entire vertex will fail.
-   *
-   * @throws IOException
-   */
-  public abstract void commitVertex() throws IOException;
-
-  /**
-   * For aborting an unsuccessful vertex's output. Note that this is invoked for
-   * vertices with a final failed state. This is called from the application
-   * master process for the entire vertex. This may be called multiple times.
-   *
-   * @param state final runstate of the vertex
-   * @throws IOException
-   */
-  public abstract void abortVertex(VertexStatus.State finalState)
-      throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
deleted file mode 100644
index 9ba2ece..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.impl;
-
-public class VertexStatus {
-
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-
-    int value;
-
-    State(int value) {
-      this.value = value;
-    }
-
-    public int getValue() {
-      return value;
-    }
-
-  };
-
-  private State runState;
-
-  public synchronized State getState() {
-    return runState;
-  }
-
-  public synchronized void setState(State state) {
-    this.runState = state;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
new file mode 100644
index 0000000..7726d7b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public interface AMInfo {
+  public ApplicationAttemptId getAppAttemptId();
+  public long getStartTime();
+  public ContainerId getContainerId();
+  public String getNodeManagerHost();
+  public int getNodeManagerPort();
+  public int getNodeManagerHttpPort();
+
+  public void setAppAttemptId(ApplicationAttemptId appAttemptId);
+  public void setStartTime(long startTime);
+  public void setContainerId(ContainerId containerId);
+  public void setNodeManagerHost(String nmHost);
+  public void setNodeManagerPort(int nmPort);
+  public void setNodeManagerHttpPort(int mnHttpPort);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
new file mode 100644
index 0000000..a2ee7e3
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+public interface TaskAttemptReport {
+  public abstract TezTaskAttemptID getTaskAttemptId();
+  public abstract TaskAttemptState getTaskAttemptState();
+  public abstract float getProgress();
+  public abstract long getStartTime();
+  public abstract long getFinishTime();
+  /** @return the shuffle finish time. Applicable only for reduce attempts */
+  public abstract long getShuffleFinishTime();
+  /** @return the sort/merge finish time. Applicable only for reduce attempts */
+  public abstract long getSortFinishTime();
+  public abstract TezCounters getCounters();
+  public abstract String getDiagnosticInfo();
+  public abstract String getStateString();
+  public abstract String getNodeManagerHost();
+  public abstract int getNodeManagerPort();
+  public abstract int getNodeManagerHttpPort();
+  public abstract ContainerId getContainerId();
+
+  public abstract void setTaskAttemptId(TezTaskAttemptID taskAttemptId);
+  public abstract void setTaskAttemptState(TaskAttemptState taskAttemptState);
+  public abstract void setProgress(float progress);
+  public abstract void setStartTime(long startTime);
+  public abstract void setFinishTime(long finishTime);
+  public abstract void setCounters(TezCounters counters);
+  public abstract void setDiagnosticInfo(String diagnosticInfo);
+  public abstract void setStateString(String stateString);
+  public abstract void setNodeManagerHost(String nmHost);
+  public abstract void setNodeManagerPort(int nmPort);
+  public abstract void setNodeManagerHttpPort(int nmHttpPort);
+  public abstract void setContainerId(ContainerId containerId);
+  
+  /** 
+   * Set the shuffle finish time. Applicable only for reduce attempts
+   * @param time the time the shuffle finished.
+   */
+  public abstract void setShuffleFinishTime(long time);
+  /** 
+   * Set the sort/merge finish time. Applicable only for reduce attempts
+   * @param time the time the shuffle finished.
+   */
+  public abstract void setSortFinishTime(long time);
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
new file mode 100644
index 0000000..068913b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+public enum TaskAttemptState {
+  NEW, 
+  STARTING, 
+  RUNNING, 
+  COMMIT_PENDING,  
+  SUCCEEDED,
+  FAILED,
+  KILLED
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
new file mode 100644
index 0000000..3a0685f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import java.util.List;
+
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+
+public interface TaskReport {
+  public abstract TezTaskID getTaskId();
+  public abstract TaskState getTaskState();
+  public abstract float getProgress();
+  public abstract long getStartTime();
+  public abstract long getFinishTime();
+  public abstract TezCounters getCounters();
+  
+  public abstract List<TezTaskAttemptID> getRunningAttemptsList();
+  public abstract TezTaskAttemptID getRunningAttempt(int index);
+  public abstract int getRunningAttemptsCount();
+  
+  public abstract TezTaskAttemptID getSuccessfulAttempt();
+  
+  public abstract List<String> getDiagnosticsList();
+  public abstract String getDiagnostics(int index);
+  public abstract int getDiagnosticsCount();
+  
+  
+  public abstract void setTaskId(TezTaskID taskId);
+  public abstract void setTaskState(TaskState taskState);
+  public abstract void setProgress(float progress);
+  public abstract void setStartTime(long startTime);
+  public abstract void setFinishTime(long finishTime);
+  public abstract void setCounters(TezCounters counters);
+  
+  public abstract void addAllRunningAttempts(List<TezTaskAttemptID> taskAttempts);
+  public abstract void addRunningAttempt(TezTaskAttemptID taskAttempt);
+  public abstract void removeRunningAttempt(int index);
+  public abstract void clearRunningAttempts();
+  
+  public abstract void setSuccessfulAttempt(TezTaskAttemptID taskAttempt);
+  public abstract void addAllDiagnostics(List<String> diagnostics);
+  public abstract void addDiagnostics(String diagnostics);
+  public abstract void removeDiagnostics(int index);
+  public abstract void clearDiagnostics();
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
new file mode 100644
index 0000000..c6b8767
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+public enum TaskState {
+  NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
deleted file mode 100644
index d57754a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-public interface AMInfo {
-  public ApplicationAttemptId getAppAttemptId();
-  public long getStartTime();
-  public ContainerId getContainerId();
-  public String getNodeManagerHost();
-  public int getNodeManagerPort();
-  public int getNodeManagerHttpPort();
-
-  public void setAppAttemptId(ApplicationAttemptId appAttemptId);
-  public void setStartTime(long startTime);
-  public void setContainerId(ContainerId containerId);
-  public void setNodeManagerHost(String nmHost);
-  public void setNodeManagerPort(int nmPort);
-  public void setNodeManagerHttpPort(int mnHttpPort);
-}
\ No newline at end of file


Mime
View raw message