tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject svn commit: r1469642 [7/36] - in /incubator/tez/branches/TEZ-1: ./ example_jobs/ example_jobs/sampleInput/ example_jobs/wc_mr_6m_1r/ example_jobs/wc_mrr_6m_3r_3r/ ljr_helper/ tez-common/ tez-common/src/ tez-common/src/main/ tez-common/src/main/java/ te...
Date Thu, 18 Apr 2013 23:54:28 GMT
Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.counters;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class LimitExceededException extends RuntimeException {
+
+  private static final long serialVersionUID = 1L;
+
+  public LimitExceededException(String msg) {
+    super(msg);
+  }
+
+  // Only allows chaining of related exceptions
+  public LimitExceededException(LimitExceededException cause) {
+    super(cause);
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.counters;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.TezJobConfig;
+
+@InterfaceAudience.Private
+public class Limits {
+
+  static final Configuration conf = new Configuration();
+  public static final int GROUP_NAME_MAX =
+      conf.getInt(TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY, 
+          TezJobConfig.COUNTER_GROUP_NAME_MAX_DEFAULT);
+  public static final int COUNTER_NAME_MAX =
+      conf.getInt(TezJobConfig.COUNTER_NAME_MAX_KEY, 
+          TezJobConfig.COUNTER_NAME_MAX_DEFAULT);
+  public static final int GROUPS_MAX =
+      conf.getInt(TezJobConfig.COUNTER_GROUPS_MAX_KEY, 
+          TezJobConfig.COUNTER_GROUPS_MAX_DEFAULT);
+  public static final int COUNTERS_MAX =
+      conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
+          COUNTERS_MAX_DEFAULT);
+
+  private int totalCounters;
+  private LimitExceededException firstViolation;
+
+  public static String filterName(String name, int maxLen) {
+    return name.length() > maxLen ? name.substring(0, maxLen - 1) : name;
+  }
+
+  public static String filterCounterName(String name) {
+    return filterName(name, COUNTER_NAME_MAX);
+  }
+
+  public static String filterGroupName(String name) {
+    return filterName(name, GROUP_NAME_MAX);
+  }
+
+  public synchronized void checkCounters(int size) {
+    if (firstViolation != null) {
+      throw new LimitExceededException(firstViolation);
+    }
+    if (size > COUNTERS_MAX) {
+      firstViolation = new LimitExceededException("Too many counters: "+ size +
+                                                  " max="+ COUNTERS_MAX);
+      throw firstViolation;
+    }
+  }
+
+  public synchronized void incrCounters() {
+    checkCounters(totalCounters + 1);
+    ++totalCounters;
+  }
+
+  public synchronized void checkGroups(int size) {
+    if (firstViolation != null) {
+      throw new LimitExceededException(firstViolation);
+    }
+    if (size > GROUPS_MAX) {
+      firstViolation = new LimitExceededException("Too many counter groups: "+
+                                                  size +" max="+ GROUPS_MAX);
+    }
+  }
+
+  public synchronized LimitExceededException violation() {
+    return firstViolation;
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.counters;
+
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.util.MissingResourceException;
+
+/**
+ * Helper class to handle resource bundles in a saner way
+ */
+public class ResourceBundles {
+
+  /**
+   * Get a resource bundle
+   * @param bundleName of the resource
+   * @return the resource bundle
+   * @throws MissingResourceException
+   */
+  public static ResourceBundle getBundle(String bundleName) {
+    return ResourceBundle.getBundle(bundleName.replace('$', '_'),
+        Locale.getDefault(), Thread.currentThread().getContextClassLoader());
+  }
+
+  /**
+   * Get a resource given bundle name and key
+   * @param <T> type of the resource
+   * @param bundleName name of the resource bundle
+   * @param key to lookup the resource
+   * @param suffix for the key to lookup
+   * @param defaultValue of the resource
+   * @return the resource or the defaultValue
+   * @throws ClassCastException if the resource found doesn't match T
+   */
+  @SuppressWarnings("unchecked")
+  public static synchronized <T> T getValue(String bundleName, String key,
+                                            String suffix, T defaultValue) {
+    T value;
+    try {
+      ResourceBundle bundle = getBundle(bundleName);
+      value = (T) bundle.getObject(getLookupKey(key, suffix));
+    }
+    catch (Exception e) {
+      return defaultValue;
+    }
+    return value == null ? defaultValue : value;
+  }
+
+  private static String getLookupKey(String key, String suffix) {
+    if (suffix == null || suffix.isEmpty()) return key;
+    return key + suffix;
+  }
+
+  /**
+   * Get the counter group display name
+   * @param group the group name to lookup
+   * @param defaultValue of the group
+   * @return the group display name
+   */
+  public static String getCounterGroupName(String group, String defaultValue) {
+    return getValue(group, "CounterGroupName", "", defaultValue);
+  }
+
+  /**
+   * Get the counter display name
+   * @param group the counter group name for the counter
+   * @param counter the counter name to lookup
+   * @param defaultValue of the counter
+   * @return the counter display name
+   */
+  public static String getCounterName(String group, String counter,
+                                      String defaultValue) {
+    return getValue(group, counter, ".name", defaultValue);
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.counters;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+// TODO TEZAM5 For MR compatibility, a conversion from tez.TaskCounters to
+// mapreduce.TaskCounters will likely be required somewhere.
+// Similarly for FileSystemCounters and others.
+
+// Counters used by Task classes
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum TaskCounter {
+  MAP_INPUT_RECORDS, 
+  MAP_OUTPUT_RECORDS,
+  MAP_SKIPPED_RECORDS,
+  MAP_OUTPUT_BYTES,
+  MAP_OUTPUT_MATERIALIZED_BYTES,
+  SPLIT_RAW_BYTES,
+  COMBINE_INPUT_RECORDS,
+  COMBINE_OUTPUT_RECORDS,
+  REDUCE_INPUT_GROUPS,
+  REDUCE_SHUFFLE_BYTES,
+  REDUCE_INPUT_RECORDS,
+  REDUCE_OUTPUT_RECORDS,
+  REDUCE_SKIPPED_GROUPS,
+  REDUCE_SKIPPED_RECORDS,
+  SPILLED_RECORDS,
+  SHUFFLED_MAPS, 
+  FAILED_SHUFFLE,
+  MERGED_MAP_OUTPUTS,
+  GC_TIME_MILLIS,
+  CPU_MILLISECONDS,
+  PHYSICAL_MEMORY_BYTES,
+  VIRTUAL_MEMORY_BYTES,
+  COMMITTED_HEAP_BYTES,
+  
+  INPUT_RECORDS, 
+  OUTPUT_RECORDS,
+  SKIPPED_RECORDS,
+  OUTPUT_BYTES,
+  OUTPUT_MATERIALIZED_BYTES,
+  INPUT_GROUPS,
+  SHUFFLE_BYTES,
+  SHUFFLED_TASKS, 
+  MERGED_TASK_OUTPUTS,
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.counters;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A named counter that tracks the progress of a map/reduce job.
+ *
+ * <p><code>Counters</code> represent global counters, defined either by the
+ * Map-Reduce framework or applications. Each <code>Counter</code> is named by
+ * an {@link Enum} and has a long for the value.</p>
+ *
+ * <p><code>Counters</code> are bunched into Groups, each comprising of
+ * counters from a particular <code>Enum</code> class.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public interface TezCounter extends Writable {
+
+  /**
+   * Set the display name of the counter
+   * @param displayName of the counter
+   * @deprecated (and no-op by default)
+   */
+  @Deprecated
+  void setDisplayName(String displayName);
+
+  /**
+   * @return the name of the counter
+   */
+  String getName();
+
+  /**
+   * Get the display name of the counter.
+   * @return the user facing name of the counter
+   */
+  String getDisplayName();
+
+  /**
+   * What is the current value of this counter?
+   * @return the current value
+   */
+  long getValue();
+
+  /**
+   * Set this counter by the given value
+   * @param value the value to set
+   */
+  void setValue(long value);
+
+  /**
+   * Increment this counter by the given value
+   * @param incr the value to increase this counter by
+   */
+  void increment(long incr);
+ 
+  /**
+   * Return the underlying object if this is a facade.
+   * @return the undelying object.
+   */
+  @Private
+  TezCounter getUnderlyingCounter();
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.common.counters;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * <p><code>Counters</code> holds per job/task counters, defined either by the
+ * Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ * any {@link Enum} type.</p>
+ *
+ * <p><code>Counters</code> are bunched into {@link CounterGroup}s, each
+ * comprising of counters from a particular <code>Enum</code> class.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezCounters extends AbstractCounters<TezCounter, CounterGroup> {
+
+  // Mix framework group implementation into CounterGroup interface
+  private static class FrameworkGroupImpl<T extends Enum<T>>
+      extends FrameworkCounterGroup<T, TezCounter> implements CounterGroup {
+
+    FrameworkGroupImpl(Class<T> cls) {
+      super(cls);
+    }
+
+    @Override
+    protected FrameworkCounter<T> newCounter(T key) {
+      return new FrameworkCounter<T>(key, getName());
+    }
+
+    @Override
+    public CounterGroupBase<TezCounter> getUnderlyingGroup() {
+      return this;
+    }
+  }
+
+  // Mix generic group implementation into CounterGroup interface
+  // and provide some mandatory group factory methods.
+  private static class GenericGroup extends AbstractCounterGroup<TezCounter>
+      implements CounterGroup {
+
+    GenericGroup(String name, String displayName, Limits limits) {
+      super(name, displayName, limits);
+    }
+
+    @Override
+    protected TezCounter newCounter(String name, String displayName, long value) {
+      return new GenericCounter(name, displayName, value);
+    }
+
+    @Override
+    protected TezCounter newCounter() {
+      return new GenericCounter();
+    }
+
+    @Override
+    public CounterGroupBase<TezCounter> getUnderlyingGroup() {
+      return this;
+    }
+  }
+
+  // Mix file system group implementation into the CounterGroup interface
+  private static class FileSystemGroup extends FileSystemCounterGroup<TezCounter>
+      implements CounterGroup {
+
+    @Override
+    protected TezCounter newCounter(String scheme, FileSystemCounter key) {
+      return new FSCounter(scheme, key);
+    }
+
+    @Override
+    public CounterGroupBase<TezCounter> getUnderlyingGroup() {
+      return this;
+    }
+  }
+
+  /**
+   * Provide factory methods for counter group factory implementation.
+   * See also the GroupFactory in
+   *  {@link org.apache.hadoop.TezCounters.Counters mapred.Counters}
+   */
+  private static class GroupFactory
+      extends CounterGroupFactory<TezCounter, CounterGroup> {
+
+    @Override
+    protected <T extends Enum<T>>
+    FrameworkGroupFactory<CounterGroup>
+        newFrameworkGroupFactory(final Class<T> cls) {
+      return new FrameworkGroupFactory<CounterGroup>() {
+        @Override public CounterGroup newGroup(String name) {
+          return new FrameworkGroupImpl<T>(cls); // impl in this package
+        }
+      };
+    }
+
+    @Override
+    protected CounterGroup newGenericGroup(String name, String displayName,
+                                           Limits limits) {
+      return new GenericGroup(name, displayName, limits);
+    }
+
+    @Override
+    protected CounterGroup newFileSystemGroup() {
+      return new FileSystemGroup();
+    }
+  }
+
+  private static final GroupFactory groupFactory = new GroupFactory();
+
+  /**
+   * Default constructor
+   */
+  public TezCounters() {
+    super(groupFactory);
+  }
+
+  /**
+   * Construct the Counters object from the another counters object
+   * @param <C> the type of counter
+   * @param <G> the type of counter group
+   * @param counters the old counters object
+   */
+  public <C extends TezCounter, G extends CounterGroupBase<C>>
+  TezCounters(AbstractCounters<C, G> counters) {
+    super(counters, groupFactory);
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java (added)
+++ incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+
+//TODO EVENTUALLY Once everything is on PB, get rid of this.
+//Alternately have the PB interfaces implement Writable.
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TezContainerId implements Writable {
+
+  private ContainerId containerId;
+
+  public TezContainerId() {
+  }
+  
+  public TezContainerId(ContainerId containerId) {
+    this.containerId = containerId;
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeLong(containerId.getApplicationAttemptId().getApplicationId()
+        .getClusterTimestamp());
+    out.writeInt(containerId.getApplicationAttemptId().getApplicationId()
+        .getId());
+    out.writeInt(containerId.getApplicationAttemptId().getAttemptId());
+    out.writeInt(containerId.getId());
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    long timestamp = in.readLong();
+    int appId = in.readInt();
+    int appAttemptId = in.readInt();
+    int id = in.readInt();
+    this.containerId = BuilderUtils.newContainerId(appId, appAttemptId,
+        timestamp, id);
+  }
+
+  @Override
+  public String toString() {
+    return containerId.toString();
+  }
+  
+  public ContainerId getContainerId() {
+    return this.containerId;
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/pom.xml
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/pom.xml?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/pom.xml (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/pom.xml Thu Apr 18 23:54:18 2013
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tez</groupId>
+    <artifactId>tez</artifactId>
+    <version>0.2.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>tez-dag-api</artifactId>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/pom.xml
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.api;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DAG { // FIXME rename to Topology
+  List<Vertex> vertices;
+  List<Edge> edges;
+  
+  public DAG() {
+    this.vertices = new ArrayList<Vertex>();
+    this.edges = new ArrayList<Edge>();
+  }
+
+  public synchronized void addVertex(Vertex vertex) {
+    if (vertices.contains(vertex)) {
+      throw new IllegalArgumentException(
+          "Vertex " + vertex + " already defined!");
+    }
+    vertices.add(vertex);
+  }
+  
+  public synchronized void addEdge(Edge edge) {
+    // Sanity checks
+    if (!vertices.contains(edge.getInputVertex())) {
+      throw new IllegalArgumentException(
+          "Input vertex " + edge.getInputVertex() + " doesn't exist!");
+    }
+    if (!vertices.contains(edge.getOutputVertex())) {
+      throw new IllegalArgumentException(
+          "Output vertex " + edge.getOutputVertex() + " doesn't exist!");    
+    }
+    if (edges.contains(edge)) {
+      throw new IllegalArgumentException(
+          "Edge " + edge + " already defined!");
+    }
+    
+    // Inform the vertices
+    edge.getInputVertex().addOutputVertex(edge.getOutputVertex(), edge.getId());
+    edge.getOutputVertex().addInputVertex(edge.getInputVertex(), edge.getId());
+    
+    edges.add(edge);
+  }
+  
+  public void verify() throws TezException { // FIXME better exception
+
+    //FIXME are task resources compulsory or will the DAG AM put in a default
+    //for each vertex if not specified?
+
+  }
+    
+  // FIXME DAGConfiguration is not public API
+  public DAGConfiguration serializeDag() {
+    DAGConfiguration dagConf = new DAGConfiguration();
+    
+    dagConf.setVertices(vertices);
+    dagConf.setEdgeProperties(edges);
+    
+    for(Vertex vertex : vertices) {
+      if(vertex.getInputVertices() != null) {
+        dagConf.setInputVertices(vertex.getVertexName(), vertex.getInputVertices());
+        dagConf.setInputEdgeIds(vertex.getVertexName(), vertex.getInputEdgeIds());
+      }
+      if(vertex.getOutputVertices() != null) {
+        dagConf.setOutputVertices(vertex.getVertexName(), vertex.getOutputVertices());
+        dagConf.setOutputEdgeIds(vertex.getVertexName(), vertex.getOutputEdgeIds());
+      }
+    }
+    
+    return dagConf;
+  }
+  
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,422 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+
+// FIXME dag conf should not be public API???
+public class DAGConfiguration extends Configuration {
+
+  private static final Log LOG = LogFactory.getLog(DAGConfiguration.class);
+
+  public DAGConfiguration(Configuration conf) {
+    super(conf);
+    if (! (conf instanceof DAGConfiguration)) {
+      this.reloadConfiguration();
+    }
+  }
+
+  public DAGConfiguration() {
+    super();
+  }
+
+  public final static String DAG = "tez.dag.";
+
+  public final static String VERTEX = DAG + "vertex.";
+
+  public final static String TASK = DAG + "task.";
+
+  public final static String TASK_ATTEMPT = DAG + "attempt.";
+
+  public final static String TEZ_DAG_VERTICES = DAG + "vertices";
+
+  public final static String TEZ_DAG_EDGES = DAG + "edges";
+
+  public final static String EDGE = DAG + "edge.";
+
+  private final static String SEPARATOR = "|";
+
+  @Private
+  public void setEdgeProperties(List<Edge> edges) {
+    String[] edgeIds = new String[edges.size()];
+    for(int i=0; i<edges.size(); ++i) {
+      edgeIds[i] = edges.get(i).getId();
+    }
+    setStrings(TEZ_DAG_EDGES, edgeIds);
+    for (Edge edge : edges) {
+      setEdgeProperty(edge.getEdgeProperty(), edge.getId());
+    }
+  }
+
+  public Map<String, EdgeProperty> getEdgeProperties() {
+    String edgeIds[] = getStrings(TEZ_DAG_EDGES);
+    if (edgeIds == null) {
+      return new TreeMap<String, EdgeProperty>();
+    }
+    Map<String, EdgeProperty> edgeProperties =
+                          new HashMap<String, EdgeProperty>(edgeIds.length);
+    for(int i=0; i<edgeIds.length; ++i) {
+      edgeProperties.put(edgeIds[i], getEdgeProperty(edgeIds[i]));
+    }
+    return edgeProperties;
+  }
+
+  private void setEdgeProperty(EdgeProperty edgeProperty, String edgeId) {
+    String[] edgeStrs = new String[4];
+    edgeStrs[0] = edgeProperty.getConnectionPattern().name();
+    edgeStrs[1] = edgeProperty.getSourceType().name();
+    edgeStrs[2] = edgeProperty.inputClass;
+    edgeStrs[3] = edgeProperty.outputClass;
+
+    setStrings(EDGE + edgeId, edgeStrs);
+  }
+
+  private EdgeProperty getEdgeProperty(String edgeId) {
+    String[] edgeStr = getStrings(EDGE + edgeId);
+    assert edgeStr.length == 4;
+    return new EdgeProperty(EdgeProperty.ConnectionPattern.valueOf(edgeStr[0]),
+                             EdgeProperty.SourceType.valueOf(edgeStr[1]),
+                             edgeStr[2],
+                             edgeStr[3]);
+  }
+
+  public final static String TEZ_DAG_VERTEX_TASKS = VERTEX + "num-tasks";
+  public final static int DEFAULT_TEZ_DAG_VERTEX_TASKS  = 0;
+
+  public int getNumVertexTasks(String vertexName) {
+    return getInt(
+        TEZ_DAG_VERTEX_TASKS + "." + vertexName,
+        DEFAULT_TEZ_DAG_VERTEX_TASKS);
+  }
+
+  public void setNumVertexTasks(String vertexName, int numTasks) {
+    setInt(
+        TEZ_DAG_VERTEX_TASKS + "." + vertexName,
+        numTasks);
+  }
+
+  public final String TEZ_DAG_VERTEX_TASK_MEMORY = TASK + "memory-mb";
+
+  public final int DEFAULT_TEZ_DAG_VERTEX_TASK_MEMORY = 1024;
+
+  public int getVertexTaskMemory(String vertexName) {
+    return getInt(
+        TEZ_DAG_VERTEX_TASK_MEMORY + "." + vertexName,
+        DEFAULT_TEZ_DAG_VERTEX_TASK_MEMORY);
+  }
+
+  public void setVertexTaskMemory(String vertexName, int memory) {
+    setInt(
+        TEZ_DAG_VERTEX_TASK_MEMORY + "." + vertexName,
+        memory);
+  }
+
+  public final String TEZ_DAG_VERTEX_TASK_CPU = TASK + "cpu-vcores";
+
+  public final int DEFAULT_TEZ_DAG_VERTEX_TASK_CORES = 1;
+
+  public int getVertexTaskCores(String vertexName) {
+    return getInt(
+        TEZ_DAG_VERTEX_TASK_CPU + "." + vertexName,
+        DEFAULT_TEZ_DAG_VERTEX_TASK_CORES);
+  }
+  public void setVertexTaskCores(String vertexName, int cores) {
+    setInt(
+        TEZ_DAG_VERTEX_TASK_CPU + "." + vertexName,
+        cores);
+  }
+
+  private final String[] EMPTY = new String[0];
+
+  public String[] getVertices() {
+    String[] vertices = getStrings(TEZ_DAG_VERTICES, EMPTY);
+    return vertices == null? EMPTY : vertices;
+  }
+
+  void setVertexResource(Vertex vertex) {
+    Resource resource = vertex.getTaskResource();
+    if (resource == null) {
+      return;
+    }
+    setVertexTaskCores(vertex.getVertexName(), resource.getVirtualCores());
+    setVertexTaskMemory(vertex.getVertexName(), resource.getMemory());
+  }
+
+  public Resource getVertexResource(String vertexName) {
+    int memory = getVertexTaskMemory(vertexName);
+    int vCores = getVertexTaskCores(vertexName);
+    return BuilderUtils.newResource(memory, vCores);
+  }
+
+  // FIXME we are serializing YarnURL which is not same as serializing a URL
+  public final String TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE = TASK + "local-resource.";
+  void setVertexLocalResource(Vertex vertex) {
+    Map<String,LocalResource> lrs = vertex.getTaskLocalResources();
+    if (lrs == null) {
+      return;
+    }
+    String[] lrStrs = new String[lrs.size()];
+    int i=0;
+    for(Map.Entry<String,LocalResource> entry : lrs.entrySet()) {
+      LocalResource lr = entry.getValue();
+      try {
+        String lrStr = StringUtils.escapeString(entry.getKey(), StringUtils.ESCAPE_CHAR,
+            SEPARATOR.charAt(0))
+            + SEPARATOR
+            + StringUtils.escapeString(
+                ConverterUtils.getPathFromYarnURL(lr.getResource()).toString(),
+                StringUtils.ESCAPE_CHAR,
+                SEPARATOR.charAt(0))
+            + SEPARATOR
+            + StringUtils.escapeString(String.valueOf(lr.getSize()), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+            + SEPARATOR
+            + StringUtils.escapeString(String.valueOf(lr.getTimestamp()), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+            + SEPARATOR
+            + StringUtils.escapeString(lr.getType().name(), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+            + SEPARATOR
+            + StringUtils.escapeString(lr.getVisibility().name(), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+            + SEPARATOR
+            + StringUtils.escapeString((lr.getPattern() == null ? "" : lr.getPattern()), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
+        LOG.info("XXXX serializing local resource:"
+            + lrStr);
+        lrStrs[i++] = StringUtils.escapeString(lrStr);
+      } catch (URISyntaxException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(), lrStrs);
+  }
+
+  public Map<String, LocalResource> getVertexLocalResources(String vertexName) {
+    String[] lrStrs = StringUtils.split(get(
+        TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertexName, ""));
+    Map<String, LocalResource> localResources =
+        new TreeMap<String, LocalResource>();
+    if (lrStrs == null) {
+      return localResources;
+    }
+    LOG.info("XXXX Found " + lrStrs.length + " local resources");
+    for (String lrStr : lrStrs) {
+      LOG.info("XXXX Parsing local resource from " + lrStr);
+      String[] tokens =
+          StringUtils.split(
+              lrStr, StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
+      if (tokens.length != 6
+          && tokens.length != 7) {
+        LOG.warn("Invalid token count in serialized LocalResource"
+            + ", serializedString=" + lrStr
+            + ", tokenCount=" + tokens.length);
+      }
+      String resourceName = tokens[0];
+      LocalResource lRsrc = Records.newRecord(LocalResource.class);
+      lRsrc.setResource(ConverterUtils.getYarnUrlFromPath(
+          new Path(tokens[1])));
+      lRsrc.setSize(Long.valueOf(tokens[2]));
+      lRsrc.setTimestamp(Long.valueOf(tokens[3]));
+      lRsrc.setType(LocalResourceType.valueOf(tokens[4]));
+      lRsrc.setVisibility(LocalResourceVisibility.valueOf(
+          tokens[5]));
+      if (tokens.length == 7) {
+        lRsrc.setPattern(tokens[6]);
+      }
+      try {
+        LOG.info("XXXX Adding local resource"
+            + ", vertexName=" + vertexName
+            + ", resourceName=" + resourceName
+            + ", resourceUrl"
+            + ConverterUtils.getPathFromYarnURL(lRsrc.getResource()).toString());
+      } catch (URISyntaxException e) {
+        // Ignore
+        // FIXME
+      }
+      localResources.put(resourceName, lRsrc);
+    }
+    return localResources;
+  }
+
+  public final String TEZ_DAG_VERTEX_TASK_ENV = TASK + "env.";
+  void setVertexEnv(Vertex vertex) {
+    Map<String, String> env = vertex.getTaskEnvironment();
+    if (env == null) {
+      return;
+    }
+    String[] envStrs = new String[env.size()];
+    int i=0;
+    for(Map.Entry<String,String> entry : env.entrySet()) {
+      String envStr = entry.getKey() + SEPARATOR + entry.getValue();
+      envStrs[i++] = StringUtils.escapeString(envStr);
+    }
+    set(TEZ_DAG_VERTEX_TASK_ENV + vertex.getVertexName(),
+        StringUtils.join(",", envStrs));
+  }
+
+  public Map<String,String> getVertexEnv(String vertexName) {
+    String[] envStrs = StringUtils.split(
+        get(TEZ_DAG_VERTEX_TASK_ENV + vertexName, ""));
+    Map<String,String> env = new HashMap<String,String>();
+    if(envStrs == null) {
+      return env;
+    }
+
+    LOG.info("XXXX Found " + envStrs.length + " environment");
+    for(String envStr : envStrs ) {
+      LOG.info("XXXX Parsing env from " + envStr);
+      StringTokenizer tokenizer = new StringTokenizer (envStr, SEPARATOR);
+      String envName = tokenizer.nextToken();
+      String envValue = tokenizer.nextToken();
+      env.put(envName, envValue);
+    }
+
+    return env;
+  }
+
+  @Private
+  public void setVertices(List<Vertex> vertices) {
+    setVertices(TEZ_DAG_VERTICES, vertices);
+    for(Vertex vertex : vertices) {
+      // set num tasks
+      setNumVertexTasks(vertex.getVertexName(), vertex.getParallelism());
+      // set resource
+      setVertexResource(vertex);
+      // set localResource
+      setVertexLocalResource(vertex);
+      // set environment
+      setVertexEnv(vertex);
+      // set processor name
+      setVertexTaskModuleClassName(vertex);
+    }
+  }
+
+  public final String TEZ_DAG_VERTEX_INPUT_VERTICES = VERTEX + "input-vertices";
+  @Public
+  @Stable
+  public String[] getInputVertices(String vertexName) {
+    String[] vertices =
+        getStrings(TEZ_DAG_VERTEX_INPUT_VERTICES + "." + vertexName, EMPTY);
+    return vertices == null? EMPTY : vertices;
+  }
+  @Private
+  public void setInputVertices(String vertexName, List<Vertex> inputVertices) {
+    setVertices(TEZ_DAG_VERTEX_INPUT_VERTICES + "." + vertexName,
+        inputVertices);
+  }
+
+  private void setVertices(String key, List<Vertex> vertices) {
+    String[] verticesNames = new String[vertices.size()];
+    for (int i = 0; i < vertices.size(); ++i) {
+      verticesNames[i] = vertices.get(i).getVertexName();
+    }
+    setStrings(key, verticesNames);
+  }
+
+  public final String TEZ_DAG_VERTEX_OUTPUT_VERTICES = VERTEX
+      + "output-vertices";
+  @Public
+  @Stable
+  public String[] getOutputVertices(String vertexName) {
+    String[] vertices =
+        getStrings(TEZ_DAG_VERTEX_OUTPUT_VERTICES + "." + vertexName, EMPTY);
+    return vertices == null? EMPTY : vertices;
+  }
+  @Private
+  public void setOutputVertices(String vertexName,
+      List<Vertex> outputVertices) {
+    setVertices(TEZ_DAG_VERTEX_OUTPUT_VERTICES + "." + vertexName,
+        outputVertices);
+  }
+
+  public final String TEZ_DAG_VERTEX_INPUT_EDGES = VERTEX + "input-edges";
+  @Public
+  @Stable
+  public List<String> getInputEdgeIds(String vertexName) {
+    return getEdgeIds(TEZ_DAG_VERTEX_INPUT_EDGES + "." + vertexName);
+  }
+  public void setInputEdgeIds(String vertexName, List<String> edgeIds) {
+    setEdgeIds(TEZ_DAG_VERTEX_INPUT_EDGES + "." + vertexName, edgeIds);
+  }
+
+  public final String TEZ_DAG_VERTEX_OUTPUT_EDGES = VERTEX + "output-edges";
+  @Public
+  @Stable
+  public List<String> getOutputEdgeIds(String vertexName) {
+    return getEdgeIds(TEZ_DAG_VERTEX_OUTPUT_EDGES + "." + vertexName);
+  }
+  @Private
+  public void setOutputEdgeIds(String vertexName, List<String> edgeIds) {
+    setEdgeIds(TEZ_DAG_VERTEX_OUTPUT_EDGES + "." + vertexName, edgeIds);
+  }
+
+  private List<String> getEdgeIds(String key) {
+    String[] edgeIds = getStrings(key, EMPTY);
+    if (edgeIds == null) {
+      return new ArrayList<String>();
+    }
+    return Arrays.asList(edgeIds);
+  }
+
+  private void setEdgeIds(String key, List<String> edgeIds) {
+    setStrings(key, edgeIds.toArray(new String[]{}));
+  }
+
+  private void setVertexTaskModuleClassName(Vertex vertex) {
+    setVertexTaskModuleClassName(vertex.getVertexName(),
+        vertex.getProcessorName());
+  }
+
+  public final String TEZ_DAG_VERTEX_TASK_MODULE= VERTEX + "task-module";
+  @Private
+  public String getVertexTaskModuleClassName(String vertexName) {
+    return get(TEZ_DAG_VERTEX_TASK_MODULE + "." + vertexName);
+  }
+  @Private
+  public void setVertexTaskModuleClassName(String vertexName,
+      String taskModule) {
+    set(TEZ_DAG_VERTEX_TASK_MODULE + "." + vertexName, taskModule);
+  }
+
+  /// File used for storing location hints that are passed to the DAG
+  public static final String DAG_LOCATION_HINT_RESOURCE_FILE =
+      DAG + "location-hint-resource-file";
+  public static final String DEFAULT_DAG_LOCATION_HINT_RESOURCE_FILE =
+      "tezdaglocationhint.info";
+
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+public class DAGLocationHint implements Writable {
+
+  private Map<String, VertexLocationHint> vertexLocationHints;
+
+  public DAGLocationHint() {
+    vertexLocationHints = new TreeMap<String, VertexLocationHint>();
+  }
+
+  public Map<String, VertexLocationHint> getVertexLocationHints() {
+    return vertexLocationHints;
+  }
+
+  public VertexLocationHint getVertexLocationHint(String vertexName) {
+    return vertexLocationHints.get(vertexName);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(vertexLocationHints.size());
+    for (Entry<String, VertexLocationHint> entry :
+        vertexLocationHints.entrySet()) {
+      Text.writeString(out, entry.getKey());
+      out.writeBoolean(entry.getValue() != null);
+      if (entry.getValue() != null) {
+        entry.getValue().write(out);
+      }
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    int entryCount = in.readInt();
+    vertexLocationHints = new TreeMap<String, VertexLocationHint>();
+    for (int i = 0; i < entryCount; ++i) {
+      String vertexName = Text.readString(in);
+      if (!in.readBoolean()) {
+        vertexLocationHints.put(vertexName, null);
+      } else {
+        VertexLocationHint hint = new VertexLocationHint();
+        hint.readFields(in);
+        vertexLocationHints.put(vertexName, hint);
+      }
+    }
+  }
+
+  public static DAGLocationHint initDAGDagLocationHint(
+      String locationHintFile) throws IOException {
+    DataInput in = new DataInputStream(new FileInputStream(locationHintFile));
+    DAGLocationHint dagLocationHint = new DAGLocationHint();
+    dagLocationHint.readFields(in);
+    return dagLocationHint;
+  }
+
+  public static void writeDAGDagLocationHint(
+      DAGLocationHint dagLocationHint,
+      String locationHintFile) throws IOException {
+    DataOutput out = new DataOutputStream(new FileOutputStream(
+        locationHintFile));
+    dagLocationHint.write(out);
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Edge.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Edge.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Edge.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Edge.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.api;
+
+public class Edge{
+  
+  private final Vertex inputVertex;
+  private final Vertex outputVertex;
+  private final EdgeProperty edgeProperty;
+    
+  public Edge(Vertex inputVertex, 
+               Vertex outputVertex, 
+               EdgeProperty edgeProperty) {
+    this.inputVertex = inputVertex;
+    this.outputVertex = outputVertex;
+    this.edgeProperty = edgeProperty;
+  }
+  
+  public Vertex getInputVertex() {
+    return inputVertex;
+  }
+  
+  public Vertex getOutputVertex() {
+    return outputVertex;
+  }
+  
+  public EdgeProperty getEdgeProperty() {
+    return edgeProperty;
+  }
+  
+  /*
+   * Used to identify the edge in the configuration
+   */
+  public String getId() {
+    return String.valueOf(this.hashCode());
+  }
+ 
+  @Override
+  public String toString() {
+    return inputVertex + " -> " + outputVertex + " (" + edgeProperty + ")";
+  }
+  
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Edge.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.api;
+
+public class EdgeProperty { // FIXME rename to ChannelProperty
+  
+  public enum ConnectionPattern {
+    ONE_TO_ONE,
+    ONE_TO_ALL,
+    BIPARTITE // FIXME rename to SHUFFLE
+  }
+  
+  public enum SourceType {
+    STABLE,
+    STABLE_PERSISTED,
+    STREAMING
+  }
+  
+  ConnectionPattern connectionPattern;
+  SourceType sourceType;
+  String inputClass;
+  String outputClass;
+  
+  public EdgeProperty(ConnectionPattern connectionPattern, 
+                       SourceType sourceType,
+                       String inputClass,
+                       String outputClass) {
+    this.connectionPattern = connectionPattern;
+    this.sourceType = sourceType;
+    this.inputClass = inputClass;
+    this.outputClass = outputClass;
+  }
+  
+  // default constructor for mapreduce case
+  public EdgeProperty() {
+    this.connectionPattern = ConnectionPattern.BIPARTITE;
+    this.sourceType = SourceType.STABLE;
+    this.inputClass = "ShuffleInput.class"; //FIXME
+    this.outputClass = "SortedMapOutputBuffer.class"; //FIXME
+  }
+  
+  public ConnectionPattern getConnectionPattern() {
+    return connectionPattern;
+  }
+  
+  public SourceType getSourceType() {
+    return sourceType;
+  }
+  
+  public String getInputClass() {
+    return inputClass;
+  }
+  
+  public String getOutputClass() {
+    return outputClass;
+  }
+  
+  @Override
+  public String toString() {
+    return "{ " + connectionPattern + " : " 
+            + inputClass + " >> " + sourceType + " >> " + outputClass + " }";
+  }
+  
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class TezConfiguration extends Configuration {
+
+  public final static String TEZ_SITE_XML = "tez-site.xml";
+  public final static String TEZ_DEFAULT_XML = "tez-default.xml";
+
+  static {
+    addDefaultResource(TEZ_DEFAULT_XML);
+    addDefaultResource(TEZ_SITE_XML);
+  }
+
+  public TezConfiguration() {
+    super();
+  }
+
+  public TezConfiguration(Configuration conf) {
+    super(conf);
+  }
+
+  public static final String TEZ_PREFIX = "tez.";
+  public static final String DAG_AM_PREFIX = TEZ_PREFIX + "dag,am.";
+
+  public static final String DAG_AM_RESOURCE_MEMORY_MB = DAG_AM_PREFIX
+      + "resource.memory.mb";
+  public static final int DEFAULT_DAG_AM_RESOURCE_MEMORY_MB = 1024;
+
+  public static final String DAG_AM_RESOURCE_CPU_VCORES = DAG_AM_PREFIX
+      + "resource.cpu.vcores";
+  public static final int DEFAULT_DAG_AM_RESOURCE_CPU_VCORES = 1;
+
+  private static final String TEZ_CONF_DIR_ENV = "TEZ_CONF_DIR";
+  private static final String TEZ_HOME_ENV = "TEZ_HOME";
+
+  public static final String TEZ_APPLICATION_CLASSPATH = TEZ_PREFIX
+      + "application.classpath";
+  public static final String[] DEFAULT_TEZ_APPLICATION_CLASSPATH = {
+    TEZ_CONF_DIR_ENV,
+    TEZ_HOME_ENV + "/*",
+    TEZ_HOME_ENV + "/lib/*"
+  };
+
+  public static final String DAG_AM_PLAN_CONFIG_XML = "tez-dag.xml";
+
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,33 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api;
+
+/*
+ * Base Tez Exception
+ */
+public class TezException extends RuntimeException {
+
+  private static final long serialVersionUID = -4956339297375386184L;
+  
+  public TezException(Throwable cause) { super(cause); }
+  public TezException(String message) { super(message); }
+  public TezException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+
+public class Vertex { // FIXME rename to Task
+
+  private final String vertexName;
+  private final String processorName;
+  
+  private final int parallelism;
+  private VertexLocationHint taskLocationsHint;
+  private Resource taskResource;
+  private Map<String, LocalResource> taskLocalResources;
+  private Map<String, String> taskEnvironment;
+  
+  private final List<Vertex> inputVertices = new ArrayList<Vertex>();
+  private final List<Vertex> outputVertices = new ArrayList<Vertex>();
+  private final List<String> inputEdgeIds = new ArrayList<String>();
+  private final List<String> outputEdgeIds = new ArrayList<String>();
+  
+  public Vertex(String vertexName, String processorName, int parallelism) {
+    this.vertexName = vertexName;
+    this.processorName = processorName;
+    this.parallelism = parallelism;
+  }
+  
+  public String getVertexName() { // FIXME rename to getName()
+    return vertexName;
+  }
+  
+  public String getProcessorName() {
+    return processorName;
+  }
+
+  public int getParallelism() {
+    return parallelism;
+  }
+  
+  public void setTaskResource(Resource resource) {
+    this.taskResource = resource;
+  }
+  
+  public Resource getTaskResource() {
+    return taskResource;
+  }
+  
+  public void setTaskLocationsHint(TaskLocationHint[] locations) {
+    assert locations.length == parallelism;
+    taskLocationsHint = new VertexLocationHint(parallelism, locations);
+  }
+
+  // used internally to create parallelism location resource file
+  VertexLocationHint getTaskLocationsHint() {
+    return taskLocationsHint;
+  }
+  
+  public void setTaskLocalResources(Map<String, LocalResource> localResources) {
+    this.taskLocalResources = localResources;
+  }
+  
+  public Map<String, LocalResource> getTaskLocalResources() {
+    return taskLocalResources;
+  }
+  
+  public void setTaskEnvironment(Map<String, String> environment) {
+    this.taskEnvironment = environment;
+  }
+  
+  public Map<String, String> getTaskEnvironment() {
+    return taskEnvironment;
+  }
+  
+  @Override
+  public String toString() {
+    return "[" + vertexName + " : " + processorName + "]";
+  }
+
+  void addInputVertex(Vertex inputVertex, String edgeId) {
+    inputVertices.add(inputVertex);
+    inputEdgeIds.add(edgeId);
+  }
+
+  void addOutputVertex(Vertex outputVertex, String edgeId) {
+    outputVertices.add(outputVertex);
+    outputEdgeIds.add(edgeId);
+  }
+  
+  List<Vertex> getInputVertices() {
+    return inputVertices;
+  }
+  
+  List<Vertex> getOutputVertices() {
+    return outputVertices;
+  }
+  
+  List<String> getInputEdgeIds() {
+    return inputEdgeIds;
+  }
+
+  List<String> getOutputEdgeIds() {
+    return outputEdgeIds;
+  }
+
+  // FIXME add java opts?
+
+  // FIXME how do we support profiling? Can't profile all tasks.
+
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+public class VertexLocationHint implements Writable {
+
+  private int numTasks;
+  private TaskLocationHint[] taskLocationHints;
+
+  public VertexLocationHint() {
+    this(0);
+  }
+
+  public VertexLocationHint(int numTasks) {
+    this(numTasks, new TaskLocationHint[numTasks]);
+  }
+
+  public VertexLocationHint(int numTasks,
+      TaskLocationHint[] taskLocationHints) {
+    this.numTasks = numTasks;
+    this.taskLocationHints = taskLocationHints;
+  }
+
+  public int getNumTasks() {
+    return numTasks;
+  }
+
+  public TaskLocationHint[] getTaskLocationHints() {
+    return taskLocationHints;
+  }
+
+  public void setTaskLocationHints(TaskLocationHint[] taskLocationHints) {
+    this.taskLocationHints = taskLocationHints;
+  }
+
+  public static class TaskLocationHint implements Writable {
+
+    // Host names if any to be used
+    private String[] hosts;
+    // Rack names if any to be used
+    private String[] racks;
+
+    public TaskLocationHint() {
+      this(new String[0], new String[0]);
+    }
+
+    public TaskLocationHint(String[] hosts, String[] racks) {
+      this.hosts = hosts;
+      this.racks = racks;
+    }
+
+    public String[] getDataLocalHosts() {
+      return hosts;
+    }
+    public void setDataLocalHosts(String[] hosts) {
+      this.hosts = hosts;
+    }
+    public String[] getRacks() {
+      return racks;
+    }
+    public void setRacks(String[] racks) {
+      this.racks = racks;
+    }
+
+    private void writeStringArray(DataOutput out, String[] array)
+        throws IOException {
+      if (array == null) {
+        out.writeInt(-1);
+        return;
+      }
+      out.writeInt(array.length);
+      for (String entry : array) {
+        out.writeBoolean(entry != null);
+        if (entry != null) {
+          Text.writeString(out, entry);
+        }
+      }
+    }
+
+    private String[] readStringArray(DataInput in)
+        throws IOException {
+      int arrayLen = in.readInt();
+      if (arrayLen == -1) {
+        return null;
+      }
+      String[] array = new String[arrayLen];
+      for (int i = 0; i < arrayLen; ++i) {
+        if (!in.readBoolean()) {
+          array[i] = null;
+        } else {
+          array[i] = Text.readString(in);
+        }
+      }
+      return array;
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      writeStringArray(out, hosts);
+      writeStringArray(out, racks);
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      hosts = readStringArray(in);
+      racks = readStringArray(in);
+    }
+
+
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(numTasks);
+    for (int i = 0; i < numTasks; ++i) {
+      out.writeBoolean(taskLocationHints[i] != null);
+      if (taskLocationHints[i] != null) {
+        taskLocationHints[i].write(out);
+      }
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    numTasks = in.readInt();
+    taskLocationHints = new TaskLocationHint[numTasks];
+    for (int i = 0; i < numTasks; ++i) {
+      if (!in.readBoolean()) {
+        taskLocationHints[i] = null;
+      } else {
+        taskLocationHints[i] = new TaskLocationHint(null, null);
+        taskLocationHints[i].readFields(in);
+      }
+    }
+  }
+
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.client;
+
+public class DAGStatus {
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4),
+    KILLED(5);
+    
+    int value;
+    
+    State(int value) {
+      this.value = value;
+    }
+    
+    public int getValue() {
+      return value; 
+    }
+    
+  };
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.client;
+
+public class VertexStatus {
+
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4),
+    KILLED(5);
+
+    int value;
+
+    State(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return value;
+    }
+
+  };
+
+  private State runState;
+
+  public synchronized State getState() {
+    return runState;
+  }
+
+  public synchronized void setState(State state) {
+    this.runState = state;
+  }
+
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag/pom.xml
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/pom.xml?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/pom.xml (added)
+++ incubator/tez/branches/TEZ-1/tez-dag/pom.xml Thu Apr 18 23:54:18 2013
@@ -0,0 +1,101 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tez</groupId>
+    <artifactId>tez</artifactId>
+    <version>0.2.0-SNAPSHOT</version>
+  </parent>
+  <properties>
+    <maven.test.redirectTestOutputToFile>true
+    </maven.test.redirectTestOutputToFile>
+    <tez.component>tez-dag</tez.component>
+  </properties>
+  <artifactId>tez-dag</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-mapreduce</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-engine</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-engine-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <configuration>
+            <forkMode>always</forkMode>
+            <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
+            <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
+            <environmentVariables>
+              <MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
+            </environmentVariables>
+            <systemPropertyVariables>
+              <java.security.krb5.conf>${basedir}/src/test/resources/krb5.conf
+              </java.security.krb5.conf>
+            </systemPropertyVariables>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

Propchange: incubator/tez/branches/TEZ-1/tez-dag/pom.xml
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java?rev=1469642&view=auto
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java (added)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java Thu Apr 18 23:54:18 2013
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+
+public class CustomOutputCommitter extends OutputCommitter {
+
+  public static final String JOB_SETUP_FILE_NAME = "_job_setup";
+  public static final String JOB_COMMIT_FILE_NAME = "_job_commit";
+  public static final String JOB_ABORT_FILE_NAME = "_job_abort";
+  public static final String TASK_SETUP_FILE_NAME = "_task_setup";
+  public static final String TASK_ABORT_FILE_NAME = "_task_abort";
+  public static final String TASK_COMMIT_FILE_NAME = "_task_commit";
+
+  @Override
+  public void setupJob(JobContext jobContext) throws IOException {
+    writeFile(jobContext.getJobConf(), JOB_SETUP_FILE_NAME);
+  }
+
+  @Override
+  public void commitJob(JobContext jobContext) throws IOException {
+    super.commitJob(jobContext);
+    writeFile(jobContext.getJobConf(), JOB_COMMIT_FILE_NAME);
+  }
+
+  @Override
+  public void abortJob(JobContext jobContext, int status) 
+  throws IOException {
+    super.abortJob(jobContext, status);
+    writeFile(jobContext.getJobConf(), JOB_ABORT_FILE_NAME);
+  }
+  
+  @Override
+  public void setupTask(TaskAttemptContext taskContext) throws IOException {
+    writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
+  }
+
+  @Override
+  public boolean needsTaskCommit(TaskAttemptContext taskContext)
+      throws IOException {
+    return true;
+  }
+
+  @Override
+  public void commitTask(TaskAttemptContext taskContext) throws IOException {
+    writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
+  }
+
+  @Override
+  public void abortTask(TaskAttemptContext taskContext) throws IOException {
+    writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
+  }
+
+  private void writeFile(JobConf conf , String filename) throws IOException {
+    System.out.println("writing file ----" + filename);
+    Path outputPath = FileOutputFormat.getOutputPath(conf);
+    FileSystem fs = outputPath.getFileSystem(conf);
+    fs.create(new Path(outputPath, filename)).close();
+  }
+}

Propchange: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message