tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1457129 [4/38] - in /incubator/tez: ./ tez-ampool/ tez-ampool/src/ tez-ampool/src/main/ tez-ampool/src/main/bin/ tez-ampool/src/main/conf/ tez-ampool/src/main/java/ tez-ampool/src/main/java/org/ tez-ampool/src/main/java/org/apache/ tez-amp...
Date Fri, 15 Mar 2013 21:26:48 GMT
Added: incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java (added)
+++ incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.ampool.rest;
+
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.ampool.AMContext;
+import org.apache.tez.ampool.manager.AMPoolManager;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+
+@Path("/applications")
+public class ApplicationPollService {
+
+  private static final Log LOG =
+      LogFactory.getLog(ApplicationPollService.class);
+
+  private static AMPoolManager manager;
+
+  public static void init(AMPoolManager manager) {
+    ApplicationPollService.manager = manager;
+  }
+
+  @Path("poll/{applicationAttemptId}")
+  @GET
+  @Consumes(MediaType.APPLICATION_JSON)
+  @Produces({MediaType.APPLICATION_JSON})
+  public ApplicationPollResponse pollForApplication(
+      @PathParam("applicationAttemptId") String applicationAttemptIdStr,
+      @Context HttpServletRequest req) {
+    ApplicationAttemptId applicationAttemptId;
+    LOG.info("Received a poll from launchedAM"
+        + ", appAttemptId=" + applicationAttemptIdStr);
+    try {
+      applicationAttemptId =
+          ConverterUtils.toApplicationAttemptId(applicationAttemptIdStr);
+    } catch (IllegalArgumentException e) {
+      LOG.warn("Received an invalid attempt id in the poll request"
+          + ", appAttemptId=" + applicationAttemptIdStr);
+      throw new WebApplicationException(Status.BAD_REQUEST);
+    }
+
+    AMContext amContext = null;
+    try {
+      amContext =
+          manager.getAMContext(applicationAttemptId);
+    } catch (Exception e) {
+      throw new WebApplicationException(e, Status.INTERNAL_SERVER_ERROR);
+    }
+
+    if (amContext == null
+        || amContext.getSubmissionContext() == null) {
+      throw new WebApplicationException(Status.NO_CONTENT);
+    }
+    try {
+      return convertAMContext(amContext);
+    } catch (Exception e) {
+      throw new WebApplicationException(e, Status.INTERNAL_SERVER_ERROR);
+    }
+  }
+
+  private ApplicationPollResponse convertAMContext(
+      AMContext amContext) throws Exception {
+    ApplicationSubmissionContext context =
+        amContext.getSubmissionContext();
+    Map<String, LocalResource> resources =
+        context.getAMContainerSpec().getLocalResources();
+    ApplicationPollResponse pollResponse =
+        new ApplicationPollResponse(context.getApplicationId(),
+            amContext.getApplicationSubmissionTime());
+    if (resources.containsKey(MRJobConfig.JOB_CONF_FILE)) {
+      LocalResource rsrc = resources.get(MRJobConfig.JOB_CONF_FILE);
+      pollResponse.setConfigurationFileLocation(
+          ConverterUtils.getPathFromYarnURL(rsrc.getResource()).toString());
+    }
+    if (resources.containsKey(MRJobConfig.JOB_JAR)) {
+      LocalResource rsrc = resources.get(MRJobConfig.JOB_JAR);
+      pollResponse.setApplicationJarLocation(
+          ConverterUtils.getPathFromYarnURL(rsrc.getResource()).toString());
+    }
+    return pollResponse;
+  }
+
+}

Added: incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml (added)
+++ incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+
+  <property>
+     <description>How often to poll AMPoolService for job assignment</description>
+     <name>yarn.app.mapreduce.am.lazy.polling-interval.secs</name>
+     <value>1</value>
+  </property>
+
+  <property>
+     <description>How many containers to pre-allocate after starting up</description>
+     <name>yarn.app.mapreduce.am.lazy.prealloc-container-count</name>
+     <value>0</value>
+  </property>
+
+</configuration>

Added: incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml (added)
+++ incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,114 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+
+  <property>
+    <description>Memory to use for AMPoolService AM</description>
+    <name>tez.ampool.am.master_memory</name>
+    <value>1024</value>
+  </property>
+
+  <property>
+    <description>Queue to launch AMPoolService AM</description>
+    <name>tez.ampool.am.master_queue</name>
+    <value>default</value>
+  </property>
+
+  <property>
+    <description>Port to use for AMPoolService status</description>
+    <name>tez.ampool.ws.port</name>
+    <value>12999</value>
+  </property>
+
+  <property>
+    <description>Minimum size of AM Pool</description>
+    <name>tez.ampool.am-pool-size</name>
+    <value>3</value>
+  </property>
+
+  <property>
+    <description>Maximum size of AM Pool</description>
+    <name>tez.ampool.max-am-pool-size</name>
+    <value>5</value>
+  </property>
+
+  <property>
+    <description>When to launch new AM. If true, launched after an existing AM in pool completes. Else, launched as soon as a job is assigned to an AM from the pool</description>
+    <name>tez.ampool.launch-new-am-after-app-completion</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <description>How many launch failures to account for unassigned AMs before shutting down AMPoolService</description>
+    <name>tez.ampool.max-am-launch-failures</name>
+    <value>10</value>
+  </property>
+
+  <property>
+    <description>No. of threads to use to serve the ClientRMProtocol proxy fronted by the AMPoolService</description>
+    <name>tez.ampool.rm-proxy-client.thread-count</name>
+    <value>10</value>
+  </property>
+
+  <property>
+    <description>Address on which to run the ClientRMProtocol proxy</description>
+    <name>tez.ampool.address</name>
+    <value>0.0.0.0:10030</value>
+  </property>
+
+  <property>
+    <description>Memory to use when launching the lazy MR AM</description>
+    <name>tez.ampool.mr-am.memory-allocation-mb</name>
+    <value>1536</value>
+  </property>
+
+  <property>
+    <description>Queue to which the Lazy MRAM is to be submitted to</description>
+    <name>tez.ampool.mr-am.queue-name</name>
+    <value>default</value>
+  </property>
+
+  <property>
+    <description>Comma-separated paths to job jars on DFS (optional)</description>
+    <name>tez.ampool.mr-am.job-jar-path</name>
+    <value></value>
+  </property>
+
+  <property>
+    <description>Main class for Lazy MRAM</description>
+    <name>tez.ampool.mr-am.application-master-class</name>
+    <value>org.apache.hadoop.mapreduce.v2.app2.lazy.LazyMRAppMaster</value>
+  </property>
+
+  <property>
+    <description>Local filesystem path for staging local data used by AMPoolClient/AMPoolService</description>
+    <name>tez.ampool.tmp-dir-path</name>
+    <value>/tmp/ampoolservice/</value>
+  </property>
+
+  <property>
+    <description>Path on FS used by AMPoolService to upload lazy-mr-am config</description>
+    <name>tez.ampool.am.staging-dir</name>
+    <value>/tmp/tez/ampool/staging/</value>
+  </property>
+
+</configuration>
+

Added: incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java (added)
+++ incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.ampool;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.client.YarnClient;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.tez.ampool.AMPoolService;
+import org.apache.tez.ampool.client.AMPoolClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestAMPool {
+
+  private static final Log LOG =
+      LogFactory.getLog(TestAMPool.class);
+
+  protected static MiniYARNCluster yarnCluster = null;
+  protected static Configuration conf = new YarnConfiguration();
+  protected static YarnClient yarnClient = null;
+
+  protected static String APPMASTER_JAR =
+      JarFinder.getJar(AMPoolService.class);
+
+  @BeforeClass
+  public static void setup() throws InterruptedException, IOException {
+    LOG.info("Starting up YARN cluster");
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER,
+        FifoScheduler.class, ResourceScheduler.class);
+    if (yarnCluster == null) {
+      yarnCluster = new MiniYARNCluster(TestAMPool.class.getName(),
+          1, 1, 1);
+      yarnCluster.init(conf);
+      yarnCluster.start();
+      URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
+      if (url == null) {
+        throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
+      }
+      yarnCluster.getConfig().set("yarn.application.classpath", new File(url.getPath()).getParent());
+      OutputStream os = new FileOutputStream(new File(url.getPath()));
+      yarnCluster.getConfig().writeXml(os);
+      yarnClient = new YarnClientImpl();
+      yarnClient.init(yarnCluster.getConfig());
+      yarnClient.start();
+      os.close();
+    }
+    try {
+      Thread.sleep(2000);
+    } catch (InterruptedException e) {
+      LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
+    }
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    if (yarnCluster != null) {
+      yarnCluster.stop();
+      yarnCluster = null;
+    }
+    if (yarnClient != null) {
+      yarnClient.stop();
+      yarnClient = null;
+    }
+  }
+
+  @Test
+  @Ignore
+  public void testSimpleRun() throws Exception {
+
+    String[] args = {
+        "--jar",
+        APPMASTER_JAR,
+        "--master_memory",
+        "512",
+    };
+
+    LOG.info("Initializing AMPool Client");
+    AMPoolClient client = new AMPoolClient();
+    boolean initSuccess = client.init(args);
+    Assert.assertTrue(initSuccess);
+    LOG.info("Running AMPool Client");
+    boolean result = client.run();
+
+    LOG.info("Client run completed. Result=" + result);
+    Assert.assertTrue(result);
+
+    while(true) {
+      try {
+        Thread.sleep(2000);
+      } catch (InterruptedException e) {
+        LOG.info("thread sleep interrupted. message=" + e.getMessage());
+      }
+
+      List<ApplicationReport> reports = yarnClient.getApplicationList();
+      if (reports.isEmpty()) {
+        LOG.error("Did not find any applications");
+        break;
+      }
+      FinalApplicationStatus status =
+          reports.get(0).getFinalApplicationStatus();
+      if (status == FinalApplicationStatus.UNDEFINED) {
+        LOG.info("Application is still running");
+        continue;
+      } else {
+        LOG.info("Application completed with status"
+            + status);
+        break;
+      }
+
+    }
+
+  }
+
+}

Added: incubator/tez/tez-ampool/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/resources/log4j.properties?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/resources/log4j.properties (added)
+++ incubator/tez/tez-ampool/src/test/resources/log4j.properties Fri Mar 15 21:26:36 2013
@@ -0,0 +1,19 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n

Added: incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml (added)
+++ incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+</configuration>
+

Added: incubator/tez/tez-ampool/src/test/resources/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/resources/yarn-site.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/resources/yarn-site.xml (added)
+++ incubator/tez/tez-ampool/src/test/resources/yarn-site.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+</configuration>
+

Added: incubator/tez/tez-api/.classpath
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/.classpath?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/.classpath (added)
+++ incubator/tez/tez-api/.classpath Fri Mar 15 21:26:36 2013
@@ -0,0 +1,72 @@
+<classpath>
+  <classpathentry kind="src" path="src/main/java" including="**/*.java"/>
+  <classpathentry kind="output" path="target/classes"/>
+  <classpathentry kind="var" path="M2_REPO/javax/activation/activation/1.1/activation-1.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar"/>
+  <classpathentry kind="var" path="M2_REPO/asm/asm/3.1/asm-3.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/avro/avro/1.5.3/avro-1.5.3.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.4/commons-codec-1.4.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-compress/1.4/commons-compress-1.4.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-digester/commons-digester/1.8/commons-digester-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-el/commons-el/1.0/commons-el-1.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-io/commons-io/2.1/commons-io-2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-lang/commons-lang/2.5/commons-lang-2.5.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-net/commons-net/3.1/commons-net-3.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/google/guava/guava/11.0.2/guava-11.0.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-annotations/3.0.0-SNAPSHOT/hadoop-annotations-3.0.0-SNAPSHOT.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-auth/3.0.0-SNAPSHOT/hadoop-auth-3.0.0-SNAPSHOT.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-sources.jar">
+    <attributes>
+      <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+    </attributes>
+  </classpathentry>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-sources.jar">
+    <attributes>
+      <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+    </attributes>
+  </classpathentry>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-sources.jar">
+    <attributes>
+      <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+    </attributes>
+  </classpathentry>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-core-asl/1.8.8/jackson-core-asl-1.8.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-jaxrs/1.7.1/jackson-jaxrs-1.7.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-mapper-asl/1.8.8/jackson-mapper-asl-1.8.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-xc/1.7.1/jackson-xc-1.7.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar"/>
+  <classpathentry kind="var" path="M2_REPO/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-core/1.8/jersey-core-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-json/1.8/jersey-json-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-server/1.8/jersey-server-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.6.1/jets3t-0.6.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar"/>
+  <classpathentry kind="var" path="M2_REPO/jline/jline/0.9.94/jline-0.9.94.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar"/>
+  <classpathentry kind="var" path="M2_REPO/log4j/log4j/1.2.17/log4j-1.2.17.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/jboss/netty/netty/3.2.2.Final/netty-3.2.2.Final.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/google/protobuf/protobuf-java/2.4.0a/protobuf-java-2.4.0a.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.6.1/slf4j-api-1.6.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-log4j12/1.6.1/slf4j-log4j12-1.6.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/xerial/snappy/snappy-java/1.0.3.2/snappy-java-1.0.3.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/stax/stax-api/1.0.1/stax-api-1.0.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/xmlenc/xmlenc/0.52/xmlenc-0.52.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/tukaani/xz/1.0/xz-1.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/zookeeper/zookeeper/3.4.2/zookeeper-3.4.2.jar"/>
+  <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+</classpath>
\ No newline at end of file

Added: incubator/tez/tez-api/.project
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/.project?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/.project (added)
+++ incubator/tez/tez-api/.project Fri Mar 15 21:26:36 2013
@@ -0,0 +1,13 @@
+<projectDescription>
+  <name>tez-api</name>
+  <comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
+  <projects/>
+  <buildSpec>
+    <buildCommand>
+      <name>org.eclipse.jdt.core.javabuilder</name>
+    </buildCommand>
+  </buildSpec>
+  <natures>
+    <nature>org.eclipse.jdt.core.javanature</nature>
+  </natures>
+</projectDescription>
\ No newline at end of file

Added: incubator/tez/tez-api/pom.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/pom.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/pom.xml (added)
+++ incubator/tez/tez-api/pom.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tez</groupId>
+    <artifactId>tez</artifactId>
+    <version>0.1.0</version>
+  </parent>
+  <artifactId>tez-api</artifactId>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+  </dependencies>
+</project>

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Input} represents a pipe through which an <em>tez</em> task
+ * can get input key/value pairs.
+ */
+public interface Input {
+  
+  /**
+   * Initialize <code>Input</code>.
+   * 
+   * @param conf job configuration
+   * @param master master process controlling the task
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void initialize(Configuration conf, Master master) 
+      throws IOException, InterruptedException;
+  
+  /**
+   * Check if there is another key/value pair.
+   * 
+   * @return true if a key/value pair was read
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public boolean hasNext() throws IOException, InterruptedException;
+
+  /**
+   * Get the next key.
+   * 
+   * @return the current key or null if there is no current key
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public Object getNextKey() throws IOException, InterruptedException;
+  
+  /**
+   * Get the next values.
+   * 
+   * @return the object that was read
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public Iterable<Object> getNextValues() 
+      throws IOException, InterruptedException;
+  
+  /**
+   * The current progress of the {@link Input} through its data.
+   * 
+   * @return a number between 0.0 and 1.0 that is the fraction of the data read
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public float getProgress() throws IOException, InterruptedException;
+  
+  /**
+   * Close this <code>Input</code> for future operations.
+   */
+  public void close() throws IOException;
+
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.tez.records.TezJobID;
+import org.apache.tez.records.TezTaskAttemptID;
+import org.apache.tez.records.TezTaskDependencyCompletionEventsUpdate;
+
+/**
+ * {@link Master} represents the master controlling the {@link Task}. 
+ */
+@ProtocolInfo(protocolName = "Master", protocolVersion = 1)
+public interface Master extends VersionedProtocol {
+
+  // TODO TEZAM3 This likely needs to change to be a little more generic.
+  // Many output / input relationships cannot be captured via this. The current
+  // form works primarily works for the existing MR
+
+  TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
+      TezJobID jobID, int fromEventIdx, int maxEventsToFetch,
+      TezTaskAttemptID taskAttemptId);
+
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.records.OutputContext;
+
+/**
+ * {@link Output} represents a pipe through which an <em>tez</em> task
+ * can send out outputs.
+ */
+public interface Output {
+
+  /**
+   * Initialize <code>Output</code>.
+   * 
+   * @param conf job configuration
+   * @param master master process controlling the task
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void initialize(Configuration conf, Master master) 
+      throws IOException, InterruptedException;
+
+  /** 
+   * Writes a key/value pair.
+   *
+   * @param key the key to write.
+   * @param value the value to write.
+   * @throws IOException
+   */      
+  public void write(Object key, Object value
+                             ) throws IOException, InterruptedException;
+
+  /**
+   * Returns the OutputContext for the particular <code>Output</code>. 
+   * 
+   * @return the OutputContext for this Output if it exists, otherwise null.
+   */
+  public OutputContext getOutputContext();
+  
+  /** 
+   * Close this <code>Output</code> for future operations.
+   * 
+   * @throws IOException
+   */ 
+  public void close() throws IOException, InterruptedException;
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+/**
+ * {@link Partitioner} is used by the TEZ framework to partition 
+ * output key/value pairs.
+ */
+public interface Partitioner {
+  
+  /**
+   * Get partition for given key/value.
+   * @param key key
+   * @param value value
+   * @param numPartitions number of partitions
+   * @return
+   */
+  int getPartition(Object key, Object value, int numPartitions);
+  
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Processor} represents the <em>tez</em> entity responsible for
+ * consuming {@link Input} and producing {@link Output}. 
+ */
+public interface Processor {
+  
+  /**
+   * Initialize the <code>Processor</code>.
+   * 
+   * @param conf job-configuration
+   * @param master master process controlling the task
+   * @throws IOException 
+   * @throws InterruptedException
+   */
+  public void initialize(Configuration conf, Master master) 
+      throws IOException, InterruptedException;
+  
+  /**
+   * Process input data from <code>input</code> and 
+   * send it to <code>output</code>.
+   * 
+   * @param in input
+   * @param out output
+   * @param master master process controlling the task
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void process(Input in, Output  out) 
+      throws IOException, InterruptedException;
+
+  /**
+   * Close the {@link Processor}.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void close() throws IOException, InterruptedException;
+
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Task} is the base <em>tez</em> entity which consumes 
+ * input key/value pairs through an {@link Input} pipe, 
+ * processes them via a {@link Processor} and 
+ * produces output key/value pairs for an {@link Output} pipe.
+ */
+public interface Task {
+  
+  /**
+   * Initialize the {@link Task}.
+   * 
+   * @param conf task configuration
+   * @param master master controlling the task
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void initialize(Configuration conf, Master master) 
+      throws IOException, InterruptedException;
+  
+  /**
+   * Get {@link Input} of the task.
+   * @return <code>Input</code> of the task
+   */
+  public Input getInput();
+
+  /**
+   * Get {@link Processor} of the task.
+   * @return <code>Processor</code> of the task
+   */
+  public Processor getProcessor();
+
+  /**
+   * Get {@link Output} of the task.
+   * @return <code>Output</code> of the task
+   */
+  public Output getOutput();
+
+  /**
+   * Run the {@link Task}.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void run() throws IOException, InterruptedException;
+  
+  /**
+   * Stop the {@link Task}.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void close() throws IOException, InterruptedException;
+  
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+//TODO EVENTUALLY Add more interfaces. Maybe make this an abstract class.
+/**
+ * Contains context information for Output. For example, shuffle headers, size
+ * of output, etc. TODO Ideally should be Output specific.
+ */
+
+public class OutputContext implements Writable {
+
+  public OutputContext(int shufflePort) {
+    this.shufflePort = shufflePort;
+  }
+  
+  public OutputContext() {
+  }
+
+  public int shufflePort;
+
+  public int getShufflePort() {
+    return this.shufflePort;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(shufflePort);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    shufflePort = in.readInt();
+  }
+  
+  @Override
+  public String toString() {
+    return "shufflePort: " + shufflePort;
+  }
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * This is used to track task completion events on 
+ * job tracker. 
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+// TODO TEZAM3 This needs to be more generic. Maybe some kind of a serialized
+// blob - which can be interpretted by the Input plugin.
+public class TezDependentTaskCompletionEvent implements Writable {
+  @InterfaceAudience.Public
+  @InterfaceStability.Evolving
+  // TODO EVENTUALLY - Remove TIPFAILED state ?
+  static public enum Status {FAILED, KILLED, SUCCEEDED, OBSOLETE, TIPFAILED};
+    
+  private int eventId;
+  // TODO EVENTUALLY - rename.
+  private String taskTrackerHttp;
+  private int taskRunTime; // using int since runtime is the time difference
+  private TezTaskAttemptID taskAttemptId;
+  Status status;
+  // TODO TEZAM2 Get rid of the isMap field. Job specific type information can be determined from TaskAttemptId.getTaskType
+  boolean isMap = false;
+  public static final TezDependentTaskCompletionEvent[] EMPTY_ARRAY = 
+    new TezDependentTaskCompletionEvent[0];
+  /**
+   * Default constructor for Writable.
+   *
+   */
+  public TezDependentTaskCompletionEvent(){
+    taskAttemptId = new TezTaskAttemptID();
+  }
+
+  /**
+   * Constructor. eventId should be created externally and incremented
+   * per event for each job. 
+   * @param eventId event id, event id should be unique and assigned in
+   *  incrementally, starting from 0. 
+   * @param taskId task id
+   * @param status task's status 
+   * @param taskTrackerHttp task tracker's host:port for http. 
+   */
+  public TezDependentTaskCompletionEvent(int eventId, 
+                             TezTaskAttemptID taskId,
+                             boolean isMap,
+                             Status status, 
+                             String taskTrackerHttp){
+      
+    this.taskAttemptId = taskId;
+    this.isMap = isMap;
+    this.eventId = eventId; 
+    this.status =status; 
+    this.taskTrackerHttp = taskTrackerHttp;
+  }
+  /**
+   * Returns event Id. 
+   * @return event id
+   */
+  public int getEventId() {
+    return eventId;
+  }
+  
+  /**
+   * Returns task id. 
+   * @return task id
+   */
+  public TezTaskAttemptID getTaskAttemptID() {
+    return taskAttemptId;
+  }
+  
+  /**
+   * Returns enum Status.SUCESS or Status.FAILURE.
+   * @return task tracker status
+   */
+  public Status getStatus() {
+    return status;
+  }
+  /**
+   * http location of the tasktracker where this task ran. 
+   * @return http location of tasktracker user logs
+   */
+  public String getTaskTrackerHttp() {
+    return taskTrackerHttp;
+  }
+
+  /**
+   * Returns time (in millisec) the task took to complete. 
+   */
+  public int getTaskRunTime() {
+    return taskRunTime;
+  }
+
+  /**
+   * Set the task completion time
+   * @param taskCompletionTime time (in millisec) the task took to complete
+   */
+  protected void setTaskRunTime(int taskCompletionTime) {
+    this.taskRunTime = taskCompletionTime;
+  }
+
+  /**
+   * set event Id. should be assigned incrementally starting from 0. 
+   * @param eventId
+   */
+  protected void setEventId(int eventId) {
+    this.eventId = eventId;
+  }
+
+  /**
+   * Sets task id. 
+   * @param taskId
+   */
+  protected void setTaskAttemptID(TezTaskAttemptID taskId) {
+    this.taskAttemptId = taskId;
+  }
+  
+  /**
+   * Set task status. 
+   * @param status
+   */
+  protected void setTaskStatus(Status status) {
+    this.status = status;
+  }
+  
+  /**
+   * Set task tracker http location. 
+   * @param taskTrackerHttp
+   */
+  protected void setTaskTrackerHttp(String taskTrackerHttp) {
+    this.taskTrackerHttp = taskTrackerHttp;
+  }
+    
+  @Override
+  public String toString(){
+    StringBuffer buf = new StringBuffer(); 
+    buf.append("Task Id : "); 
+    buf.append(taskAttemptId); 
+    buf.append(", Status : ");  
+    buf.append(status.name());
+    return buf.toString();
+  }
+    
+  @Override
+  public boolean equals(Object o) {
+    if(o == null)
+      return false;
+    if(o.getClass().equals(this.getClass())) {
+      TezDependentTaskCompletionEvent event = (TezDependentTaskCompletionEvent) o;
+      return this.isMap == event.isMapTask() 
+             && this.eventId == event.getEventId()
+             && this.status.equals(event.getStatus())
+             && this.taskAttemptId.equals(event.getTaskAttemptID()) 
+             && this.taskRunTime == event.getTaskRunTime()
+             && this.taskTrackerHttp.equals(event.getTaskTrackerHttp());
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return toString().hashCode(); 
+  }
+
+  public boolean isMapTask() {
+    return isMap;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    taskAttemptId.write(out);
+    out.writeBoolean(isMap);
+    WritableUtils.writeEnum(out, status);
+    WritableUtils.writeString(out, taskTrackerHttp);
+    WritableUtils.writeVInt(out, taskRunTime);
+    WritableUtils.writeVInt(out, eventId);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    taskAttemptId.readFields(in);
+    isMap = in.readBoolean();
+    status = WritableUtils.readEnum(in, Status.class);
+    taskTrackerHttp = WritableUtils.readString(in);
+    taskRunTime = WritableUtils.readVInt(in);
+    eventId = WritableUtils.readVInt(in);
+    
+  }
+  
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A general identifier, which internally stores the id
+ * as an integer. This is the super class of {@link TezJobID}, 
+ * {@link TezTaskID} and {@link TezTaskAttemptID}.
+ * 
+ * @see TezJobID
+ * @see TezTaskID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public abstract class TezID implements WritableComparable<TezID> {
+  public static final char SEPARATOR = '_';
+  protected int id;
+
+  /** constructs an ID object from the given int */
+  public TezID(int id) {
+    this.id = id;
+  }
+
+  protected TezID() {
+  }
+
+  /** returns the int which represents the identifier */
+  public int getId() {
+    return id;
+  }
+
+  @Override
+  public String toString() {
+    return String.valueOf(id);
+  }
+
+  @Override
+  public int hashCode() {
+    return id;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o)
+      return true;
+    if(o == null)
+      return false;
+    if (o.getClass() == this.getClass()) {
+      TezID that = (TezID) o;
+      return this.id == that.id;
+    }
+    else
+      return false;
+  }
+
+  /** Compare IDs by associated numbers */
+  public int compareTo(TezID that) {
+    return this.id - that.id;
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    this.id = in.readInt();
+  }
+
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(id);
+  }
+  
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Text;
+
+/**
+ * JobID represents the immutable and unique identifier for 
+ * the job. JobID consists of two parts. First part 
+ * represents the jobtracker identifier, so that jobID to jobtracker map 
+ * is defined. For cluster setup this string is the jobtracker 
+ * start time, for local setting, it is "local".
+ * Second part of the JobID is the job number. <br> 
+ * An example JobID is : 
+ * <code>job_200707121733_0003</code> , which represents the third job 
+ * running at the jobtracker started at <code>200707121733</code>. 
+ * <p>
+ * Applications should never construct or parse JobID strings, but rather 
+ * use appropriate constructors or {@link #forName(String)} method. 
+ * 
+ * @see TezTaskID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezJobID extends TezID 
+                   implements Comparable<TezID> {
+  protected static final String JOB = "job";
+  
+  // Jobid regex for various tools and framework components
+  public static final String JOBID_REGEX = 
+    JOB + SEPARATOR + "[0-9]+" + SEPARATOR + "[0-9]+";
+  
+  private final Text jtIdentifier;
+  
+  protected static final NumberFormat idFormat = NumberFormat.getInstance();
+  static {
+    idFormat.setGroupingUsed(false);
+    idFormat.setMinimumIntegerDigits(4);
+  }
+  
+  /**
+   * Constructs a JobID object 
+   * @param jtIdentifier jobTracker identifier
+   * @param id job number
+   */
+  public TezJobID(String jtIdentifier, int id) {
+    super(id);
+    this.jtIdentifier = new Text(jtIdentifier);
+  }
+  
+  public TezJobID() { 
+    jtIdentifier = new Text();
+  }
+  
+  public String getJtIdentifier() {
+    return jtIdentifier.toString();
+  }
+  
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezJobID that = (TezJobID)o;
+    return this.jtIdentifier.equals(that.jtIdentifier);
+  }
+  
+  /**Compare JobIds by first jtIdentifiers, then by job numbers*/
+  @Override
+  public int compareTo(TezID o) {
+    TezJobID that = (TezJobID)o;
+    int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier);
+    if(jtComp == 0) {
+      return this.id - that.id;
+    }
+    else return jtComp;
+  }
+  
+  /**
+   * Add the stuff after the "job" prefix to the given builder. This is useful,
+   * because the sub-ids use this substring at the start of their string.
+   * @param builder the builder to append to
+   * @return the builder that was passed in
+   */
+  public StringBuilder appendTo(StringBuilder builder) {
+    builder.append(SEPARATOR);
+    builder.append(jtIdentifier);
+    builder.append(SEPARATOR);
+    builder.append(idFormat.format(id));
+    return builder;
+  }
+
+  @Override
+  public int hashCode() {
+    return jtIdentifier.hashCode() + id;
+  }
+
+  @Override
+  public String toString() {
+    return appendTo(new StringBuilder(JOB)).toString();
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    super.readFields(in);
+    this.jtIdentifier.readFields(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    jtIdentifier.write(out);
+  }
+  
+  /** Construct a JobId object from given string 
+   * @return constructed JobId object or null if the given String is null
+   * @throws IllegalArgumentException if the given string is malformed
+   */
+  public static TezJobID forName(String str) throws IllegalArgumentException {
+    if(str == null)
+      return null;
+    try {
+      String[] parts = str.split("_");
+      if(parts.length == 3) {
+        if(parts[0].equals(JOB)) {
+          return new TezJobID(parts[1], Integer.parseInt(parts[2]));
+        }
+      }
+    }catch (Exception ex) {//fall below
+    }
+    throw new IllegalArgumentException("JobId string : " + str 
+        + " is not properly formed");
+  }
+  
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * TaskAttemptID represents the immutable and unique identifier for 
+ * a task attempt. Each task attempt is one particular instance of a Map or
+ * Reduce Task identified by its TaskID. 
+ * 
+ * TaskAttemptID consists of 2 parts. First part is the 
+ * {@link TezTaskID}, that this TaskAttemptID belongs to.
+ * Second part is the task attempt number. <br> 
+ * An example TaskAttemptID is : 
+ * <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ * zeroth task attempt for the fifth map task in the third job 
+ * running at the jobtracker started at <code>200707121733</code>.
+ * <p>
+ * Applications should never construct or parse TaskAttemptID strings
+ * , but rather use appropriate constructors or {@link #forName(String)} 
+ * method. 
+ * 
+ * @see TezJobID
+ * @see TezTaskID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskAttemptID extends TezID {
+  public static final String ATTEMPT = "attempt";
+  private TezTaskID taskId;
+  
+  /**
+   * Constructs a TaskAttemptID object from given {@link TezTaskID}.  
+   * @param taskId TaskID that this task belongs to  
+   * @param id the task attempt number
+   */
+  public TezTaskAttemptID(TezTaskID taskId, int id) {
+    super(id);
+    if(taskId == null) {
+      throw new IllegalArgumentException("taskId cannot be null");
+    }
+    this.taskId = taskId;
+  }
+  
+  /**
+   * Constructs a TaskId object from given parts.
+   * @param jtIdentifier jobTracker identifier
+   * @param jobId job number 
+   * @param type the TaskType 
+   * @param taskId taskId number
+   * @param id the task attempt number
+   */
+  public TezTaskAttemptID(String jtIdentifier, int jobId, String type, 
+                       int taskId, int id) {
+    this(new TezTaskID(jtIdentifier, jobId, type, taskId), id);
+  }
+  
+  public TezTaskAttemptID() { 
+    taskId = new TezTaskID();
+  }
+  
+  /** Returns the {@link TezJobID} object that this task attempt belongs to */
+  public TezJobID getJobID() {
+    return taskId.getJobID();
+  }
+  
+  /** Returns the {@link TezTaskID} object that this task attempt belongs to */
+  public TezTaskID getTaskID() {
+    return taskId;
+  }
+  
+  /**Returns the TaskType of the TaskAttemptID */
+  public String getTaskType() {
+    return taskId.getTaskType();
+  }
+  
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezTaskAttemptID that = (TezTaskAttemptID)o;
+    return this.taskId.equals(that.taskId);
+  }
+  
+  /**
+   * Add the unique string to the StringBuilder
+   * @param builder the builder to append ot
+   * @return the builder that was passed in.
+   */
+  protected StringBuilder appendTo(StringBuilder builder) {
+    return taskId.appendTo(builder).append(SEPARATOR).append(id);
+  }
+  
+  public static TezTaskAttemptID read(DataInput in) throws IOException {
+    TezTaskAttemptID taId = new TezTaskAttemptID();
+    taId.readFields(in);
+    return taId;
+  }
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    super.readFields(in);
+    taskId.readFields(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    taskId.write(out);
+  }
+
+  @Override
+  public int hashCode() {
+    return taskId.hashCode() * 5 + id;
+  }
+  
+  /**Compare TaskIds by first tipIds, then by task numbers. */
+  @Override
+  public int compareTo(TezID o) {
+    TezTaskAttemptID that = (TezTaskAttemptID)o;
+    int tipComp = this.taskId.compareTo(that.taskId);
+    if(tipComp == 0) {
+      return this.id - that.id;
+    }
+    else return tipComp;
+  }
+  @Override
+  public String toString() { 
+    return appendTo(new StringBuilder(ATTEMPT)).toString();
+  }
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+public class TezTaskDependencyCompletionEventsUpdate implements Writable {
+  TezDependentTaskCompletionEvent[] events;
+  boolean reset;
+
+  public TezTaskDependencyCompletionEventsUpdate() { }
+
+  public TezTaskDependencyCompletionEventsUpdate(
+      TezDependentTaskCompletionEvent[] events, boolean reset) {
+    this.events = events;
+    this.reset = reset;
+  }
+
+  public boolean shouldReset() {
+    return reset;
+  }
+
+  public TezDependentTaskCompletionEvent[] getDependentTaskCompletionEvents() {
+    return events;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeBoolean(reset);
+    out.writeInt(events.length);
+    for (TezDependentTaskCompletionEvent event : events) {
+      event.write(out);
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    reset = in.readBoolean();
+    events = new TezDependentTaskCompletionEvent[in.readInt()];
+    for (int i = 0; i < events.length; ++i) {
+      events[i] = new TezDependentTaskCompletionEvent();
+      events[i].readFields(in);
+    }
+  }
+
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableUtils;
+
+
+/**
+ * TaskID represents the immutable and unique identifier for 
+ * a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ * execute the Map or Reduce Task, each of which are uniquely indentified by
+ * their TaskAttemptID.
+ * 
+ * TaskID consists of 3 parts. First part is the {@link TezJobID}, that this 
+ * TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ * representing whether the task is a map task or a reduce task. 
+ * And the third part is the task number. <br> 
+ * An example TaskID is : 
+ * <code>task_200707121733_0003_m_000005</code> , which represents the
+ * fifth map task in the third job running at the jobtracker 
+ * started at <code>200707121733</code>. 
+ * <p>
+ * Applications should never construct or parse TaskID strings
+ * , but rather use appropriate constructors or {@link #forName(String)} 
+ * method. 
+ * 
+ * @see TezJobID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskID extends TezID {
+  public static final String TASK = "task";
+  protected static final NumberFormat idFormat = NumberFormat.getInstance();
+  static {
+    idFormat.setGroupingUsed(false);
+    idFormat.setMinimumIntegerDigits(6);
+  }
+  
+  private TezJobID jobId;
+  private String type;
+  
+  /**
+   * Constructs a TaskID object from given {@link TezJobID}.  
+   * @param jobId JobID that this tip belongs to 
+   * @param type the {@link TezTaskType} of the task 
+   * @param id the tip number
+   */
+  public TezTaskID(TezJobID jobId, String type, int id) {
+    super(id);
+    if(jobId == null) {
+      throw new IllegalArgumentException("jobId cannot be null");
+    }
+    this.jobId = jobId;
+    this.type = type;
+  }
+  
+  /**
+   * Constructs a TaskInProgressId object from given parts.
+   * @param jtIdentifier jobTracker identifier
+   * @param jobId job number 
+   * @param type the TaskType 
+   * @param id the tip number
+   */
+  public TezTaskID(String jtIdentifier, int jobId, String type, int id) {
+    this(new TezJobID(jtIdentifier, jobId), type, id);
+  }
+  
+  public TezTaskID() { 
+    jobId = new TezJobID();
+  }
+  
+  /** Returns the {@link TezJobID} object that this tip belongs to */
+  public TezJobID getJobID() {
+    return jobId;
+  }
+
+  /**
+   * Get the type of the task
+   */
+  public String getTaskType() {
+    return type;
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    super.readFields(in);
+    jobId.readFields(in);
+    type = WritableUtils.readString(in);
+//    String typeClassName = WritableUtils.readString(in);
+//    String enumVal = WritableUtils.readString(in);
+//    
+//    // TODO EVENTUALLY Move this into RelcetionUtils
+//    try {
+//      Class<?> cls = Class.forName(typeClassName);
+//      if (TezTaskType.class.isAssignableFrom(cls)) {
+//        Method m = cls.getMethod("valueOf", String.class);
+//        m.setAccessible(true);
+//        type = (TezTaskType) m.invoke(null, enumVal);
+//
+//      } else {
+//        throw new RuntimeException("Type: + " + typeClassName
+//            + " should be a subclass of " + TezTaskType.class.getName());
+//      }
+//    } catch (ClassNotFoundException e) {
+//      throw new RuntimeException("Unable to load typeClass: " + typeClassName,
+//          e);
+//    } catch (Exception e) {
+//      throw new RuntimeException("Failed to create instance of: "
+//          + typeClassName, e);
+//    }
+  }
+
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    jobId.write(out);
+    WritableUtils.writeString(out, type);
+//    WritableUtils.writeString(out, type.getClass().getName());
+//    // TODO Maybe define a method for default instantiation.
+//    WritableUtils.writeString(out, type.toSerializedString());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezTaskID that = (TezTaskID)o;
+    return this.type.equals(that.type) && this.jobId.equals(that.jobId);
+  }
+
+  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+  @Override
+  public int compareTo(TezID o) {
+    TezTaskID that = (TezTaskID)o;
+    int jobComp = this.jobId.compareTo(that.jobId);
+    if(jobComp == 0) {
+      if(this.type == that.type) {
+        return this.id - that.id;
+      }
+      else {
+        return this.type.toString().compareTo(that.type.toString());
+      }
+    }
+    else return jobComp;
+  }
+  @Override
+  public String toString() { 
+    return appendTo(new StringBuilder(TASK)).toString();
+  }
+
+  /**
+   * Add the unique string to the given builder.
+   * @param builder the builder to append to
+   * @return the builder that was passed in
+   */
+  protected StringBuilder appendTo(StringBuilder builder) {
+    return jobId.appendTo(builder).
+                 append(SEPARATOR).
+                 append(type.toString()).
+                 append(SEPARATOR).
+                 append(idFormat.format(id));
+  }
+  
+  @Override
+  public int hashCode() {
+    return jobId.hashCode() * 524287 + id;
+  }
+  
+}

Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,26 @@
+package org.apache.tez.records;
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with this
+// * work for additional information regarding copyright ownership. The ASF
+// * licenses this file to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// * 
+// * http://www.apache.org/licenses/LICENSE-2.0
+// * 
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// * License for the specific language governing permissions and limitations under
+// * the License.
+// */
+//
+//package org.apache.tez.records;
+//
+//
+//public interface TezTaskType {
+//  
+//  public String toSerializedString();
+//
+//}

Added: incubator/tez/tez-common/.classpath
URL: http://svn.apache.org/viewvc/incubator/tez/tez-common/.classpath?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-common/.classpath (added)
+++ incubator/tez/tez-common/.classpath Fri Mar 15 21:26:36 2013
@@ -0,0 +1,73 @@
+<classpath>
+  <classpathentry kind="src" path="src/main/java" including="**/*.java"/>
+  <classpathentry kind="output" path="target/classes"/>
+  <classpathentry kind="var" path="M2_REPO/javax/activation/activation/1.1/activation-1.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar"/>
+  <classpathentry kind="var" path="M2_REPO/asm/asm/3.1/asm-3.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/avro/avro/1.5.3/avro-1.5.3.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.4/commons-codec-1.4.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-compress/1.4/commons-compress-1.4.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-digester/commons-digester/1.8/commons-digester-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-el/commons-el/1.0/commons-el-1.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-io/commons-io/2.1/commons-io-2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-lang/commons-lang/2.5/commons-lang-2.5.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/commons-net/commons-net/3.1/commons-net-3.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/google/guava/guava/11.0.2/guava-11.0.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-annotations/3.0.0-SNAPSHOT/hadoop-annotations-3.0.0-SNAPSHOT.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-auth/3.0.0-SNAPSHOT/hadoop-auth-3.0.0-SNAPSHOT.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-sources.jar">
+    <attributes>
+      <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+    </attributes>
+  </classpathentry>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-sources.jar">
+    <attributes>
+      <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+    </attributes>
+  </classpathentry>
+  <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-sources.jar">
+    <attributes>
+      <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+    </attributes>
+  </classpathentry>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-core-asl/1.8.8/jackson-core-asl-1.8.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-jaxrs/1.7.1/jackson-jaxrs-1.7.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-mapper-asl/1.8.8/jackson-mapper-asl-1.8.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-xc/1.7.1/jackson-xc-1.7.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar"/>
+  <classpathentry kind="var" path="M2_REPO/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-core/1.8/jersey-core-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-json/1.8/jersey-json-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-server/1.8/jersey-server-1.8.jar"/>
+  <classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.6.1/jets3t-0.6.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar"/>
+  <classpathentry kind="var" path="M2_REPO/jline/jline/0.9.94/jline-0.9.94.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar"/>
+  <classpathentry kind="var" path="M2_REPO/log4j/log4j/1.2.17/log4j-1.2.17.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/jboss/netty/netty/3.2.2.Final/netty-3.2.2.Final.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar"/>
+  <classpathentry kind="var" path="M2_REPO/com/google/protobuf/protobuf-java/2.4.0a/protobuf-java-2.4.0a.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.6.1/slf4j-api-1.6.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-log4j12/1.6.1/slf4j-log4j12-1.6.1.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/xerial/snappy/snappy-java/1.0.3.2/snappy-java-1.0.3.2.jar"/>
+  <classpathentry kind="var" path="M2_REPO/stax/stax-api/1.0.1/stax-api-1.0.1.jar"/>
+  <classpathentry kind="src" path="/tez-api"/>
+  <classpathentry kind="var" path="M2_REPO/xmlenc/xmlenc/0.52/xmlenc-0.52.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/tukaani/xz/1.0/xz-1.0.jar"/>
+  <classpathentry kind="var" path="M2_REPO/org/apache/zookeeper/zookeeper/3.4.2/zookeeper-3.4.2.jar"/>
+  <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+</classpath>
\ No newline at end of file

Added: incubator/tez/tez-common/.project
URL: http://svn.apache.org/viewvc/incubator/tez/tez-common/.project?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-common/.project (added)
+++ incubator/tez/tez-common/.project Fri Mar 15 21:26:36 2013
@@ -0,0 +1,15 @@
+<projectDescription>
+  <name>tez-common</name>
+  <comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
+  <projects>
+    <project>tez-api</project>
+  </projects>
+  <buildSpec>
+    <buildCommand>
+      <name>org.eclipse.jdt.core.javabuilder</name>
+    </buildCommand>
+  </buildSpec>
+  <natures>
+    <nature>org.eclipse.jdt.core.javanature</nature>
+  </natures>
+</projectDescription>
\ No newline at end of file

Added: incubator/tez/tez-common/pom.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-common/pom.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-common/pom.xml (added)
+++ incubator/tez/tez-common/pom.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tez</groupId>
+    <artifactId>tez</artifactId>
+    <version>0.1.0</version>
+  </parent>
+  <artifactId>tez-common</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+    </dependency>
+  </dependencies>
+</project>



Mime
View raw message