tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2949. Allow duplicate dag names within session for Tez. (hitesh)
Date Fri, 20 Nov 2015 21:39:22 GMT
Repository: tez
Updated Branches:
  refs/heads/master 4561b8252 -> 9024cade5


TEZ-2949. Allow duplicate dag names within session for Tez. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9024cade
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9024cade
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9024cade

Branch: refs/heads/master
Commit: 9024cade51d4ad4ea8e2b54e27e8fac6664009a8
Parents: 4561b82
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Nov 20 13:39:05 2015 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Nov 20 13:39:05 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  4 --
 .../org/apache/tez/dag/app/RecoveryParser.java  |  3 --
 .../apache/tez/dag/app/TestRecoveryParser.java  |  1 -
 .../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 52 --------------------
 .../org/apache/tez/test/TestDAGRecovery.java    |  8 ---
 6 files changed, 2 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9644a1e..223dcc1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES
   TEZ-2948. Stop using dagName in the dagComplete notification to TaskCommunicators.
 
 ALL CHANGES:
+  TEZ-2949. Allow duplicate dag names within session for Tez.
   TEZ-2952. NPE in TestOnFileUnorderedKVOutput
   TEZ-2480. Exception when closing output is ignored.
   TEZ-2944. NPE in TestProcessorContext.
@@ -259,6 +260,7 @@ INCOMPATIBLE CHANGES
   TEZ-2679. Admin forms of launch env settings
 
 ALL CHANGES
+  TEZ-2949. Allow duplicate dag names within session for Tez.
   TEZ-2923. Tez Live UI counters view empty for vertices, tasks, attempts
   TEZ-2935. Add MR slow start translation for ShuffleVertexManager
   TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely

http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 9c09f95..2c50264 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2298,10 +2298,6 @@ public class DAGAppMaster extends AbstractService {
       throws TezException {
     long submitTime = this.clock.getTime();
     this.appName = dagPlan.getName();
-    if (dagNames.contains(dagPlan.getName())) {
-      throw new TezException("Duplicate dag name '" + dagPlan.getName() + "'");
-    }
-    dagNames.add(dagPlan.getName());
 
     // /////////////////// Create the job itself.
     DAG newDAG = createDAG(dagPlan);

http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
index 8ce835b..046dbd9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java
@@ -323,7 +323,6 @@ public class RecoveryParser {
   static class DAGSummaryData {
 
     final TezDAGID dagId;
-    String dagName;
     boolean completed = false;
     boolean dagCommitCompleted = true;
     DAGState dagState;
@@ -346,7 +345,6 @@ public class RecoveryParser {
           completed = false;
           DAGSubmittedEvent dagSubmittedEvent = new DAGSubmittedEvent();
           dagSubmittedEvent.fromSummaryProtoStream(proto);
-          dagName = dagSubmittedEvent.getDAGName();
           break;
         case DAG_FINISHED:
           completed = true;
@@ -543,7 +541,6 @@ public class RecoveryParser {
     // Set counter for next set of DAGs & update dagNames Set in DAGAppMaster
     dagAppMaster.setDAGCounter(dagCounter);
     for (DAGSummaryData dagSummaryData: dagSummaryDataMap.values()){
-      dagAppMaster.dagNames.add(dagSummaryData.dagName);
       dagAppMaster.dagIDs.add(dagSummaryData.dagId.toString());
     }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
index e261df6..be67cb2 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
@@ -70,7 +70,6 @@ public class TestRecoveryParser {
     this.recoveryPath = new Path(TEST_ROOT_DIR + "/" + appId + "/recovery");
     this.localFS.delete(new Path(TEST_ROOT_DIR), true);
     mockAppMaster = mock(DAGAppMaster.class);
-    mockAppMaster.dagNames = new HashSet<String>();
     mockAppMaster.dagIDs = new HashSet<String>();
     when(mockAppMaster.getConfig()).thenReturn(new Configuration());
     mockDAGImpl = mock(DAGImpl.class);

http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index db1b1e1..92f70c5 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -20,8 +20,6 @@ package org.apache.tez.mapreduce;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.IOException;
@@ -222,56 +220,6 @@ public class TestMRRJobsDAGApi {
     tezSession.stop();
   }
 
-  @Test(timeout = 100000)
-  public void testMultipleDAGsWithDuplicateName() throws TezException, IOException,
-      InterruptedException {
-    TezClient tezSession = null;
-    try {
-      TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
-      Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
-          .nextInt(100000))));
-      remoteFs.mkdirs(remoteStagingDir);
-      tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
-      tezSession = TezClient.create("OrderedWordCountSession", tezConf, true);
-      tezSession.start();
-
-      SleepProcessorConfig spConf = new SleepProcessorConfig(1);
-      for (int dagIndex = 1; dagIndex <= 2; dagIndex++) {
-        DAG dag = DAG.create("TezSleepProcessor");
-        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
-                SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
-            Resource.newInstance(1024, 1));
-        dag.addVertex(vertex);
-
-        DAGClient dagClient = null;
-        try {
-          dagClient = tezSession.submitDAG(dag);
-          if (dagIndex > 1) {
-            fail("Should fail due to duplicate dag name for dagIndex: " + dagIndex);
-          }
-        } catch (TezException tex) {
-          if (dagIndex > 1) {
-            assertTrue(tex.getMessage().contains("Duplicate dag name "));
-            continue;
-          }
-          fail("DuplicateDAGName exception thrown for 1st DAG submission");
-        }
-        DAGStatus dagStatus = dagClient.getDAGStatus(null);
-        while (!dagStatus.isCompleted()) {
-          LOG.debug("Waiting for job to complete. Sleeping for 500ms." + " Current state:
"
-              + dagStatus.getState());
-          Thread.sleep(500l);
-          dagStatus = dagClient.getDAGStatus(null);
-        }
-      }
-    } finally {
-      if (tezSession != null) {
-        tezSession.stop();
-      }
-    }
-  }
-  
-  
   @Test(timeout = 60000)
   public void testNonDefaultFSStagingDir() throws Exception {
     SleepProcessorConfig spConf = new SleepProcessorConfig(1);

http://git-wip-us.apache.org/repos/asf/tez/blob/9024cade/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
index 74c2727..778825b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
@@ -238,14 +238,6 @@ public class TestDAGRecovery {
     runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
 
     verifyRecoveryLog();
-
-    // it should fail if submitting same dags in recovery mode (TEZ-1064)
-    try {
-      DAGClient dagClient = tezSession.submitDAG(dag);
-      Assert.fail("Expected DAG submit to fail on duplicate dag name");
-    } catch (TezException e) {
-      Assert.assertTrue(e.getMessage().contains("Duplicate dag name"));
-    }
   }
 
   @Test(timeout=120000)


Mime
View raw message