tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject tez git commit: TEZ-3017. HistoryACLManager does not have a close method for cleanup (bikas) (cherry picked from commit 9816a49bba0c89651f9b5bb66ac20f4eb0364ef2)
Date Tue, 29 Dec 2015 09:57:49 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 73d5ce6e2 -> 9880c414c


TEZ-3017. HistoryACLManager does not have a close method for cleanup (bikas)
(cherry picked from commit 9816a49bba0c89651f9b5bb66ac20f4eb0364ef2)

Conflicts:
	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9880c414
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9880c414
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9880c414

Branch: refs/heads/branch-0.7
Commit: 9880c414c3cbd3bb5c57807adba6a9dbebcf621f
Parents: 73d5ce6
Author: Bikas Saha <bikas@apache.org>
Authored: Tue Dec 29 01:54:00 2015 -0800
Committer: Bikas Saha <bikas@apache.org>
Committed: Tue Dec 29 01:57:33 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                              |  1 +
 .../src/main/java/org/apache/tez/client/TezClient.java   |  4 ++++
 .../tez/common/security/HistoryACLPolicyManager.java     |  5 ++++-
 .../test/java/org/apache/tez/client/TestTezClient.java   |  6 +++++-
 .../dag/history/ats/acls/ATSHistoryACLPolicyManager.java | 11 ++++++++---
 .../tez/dag/history/ats/acls/TestATSHistoryWithACLs.java |  9 ++++++---
 .../history/logging/ats/ATSHistoryLoggingService.java    |  3 +++
 7 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3a5ac29..4f8897e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES
   TEZ-2949. Allow duplicate dag names within session for Tez.
 
 ALL CHANGES
+  TEZ-3017. HistoryACLManager does not have a close method for cleanup
   TEZ-2914. Ability to limit vertex concurrency
   TEZ-3011. Link Vertex Name in Dag Tasks/Task Attempts to Vertex
   TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated for DefaultSorter with multiple partitions.

http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 692e11d..3a857a0 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -524,6 +524,10 @@ public class TezClient {
    */
   public synchronized void stop() throws TezException, IOException {
     try {
+      if (historyACLPolicyManager != null) {
+        historyACLPolicyManager.close();
+      }
+
       if (sessionStarted) {
         LOG.info("Shutting down Tez Session"
             + ", sessionName=" + clientName

http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
index dea89cc..fc0f57c 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java
@@ -69,5 +69,8 @@ public interface HistoryACLPolicyManager extends Configurable {
 
   public void updateTimelineEntityDomain(Object timelineEntity, String domainId);
 
-
+  /**
+   * Call this to stop and clean up
+   */
+  public void close();
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 3ad95dd..2cd6a1a 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.tez.common.counters.LimitExceededException;
 import org.apache.tez.common.counters.Limits;
 import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.common.security.HistoryACLPolicyManager;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.PreWarmVertex;
 import org.apache.tez.dag.api.ProcessorDescriptor;
@@ -156,10 +157,12 @@ public class TestTezClient {
         LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
     
     TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
-    
+    HistoryACLPolicyManager mockAcl = mock(HistoryACLPolicyManager.class);
+
     ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
     when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
     .thenReturn(YarnApplicationState.RUNNING);
+    client.setUpHistoryAclManager(mockAcl);
     client.start();
     verify(client.mockYarnClient, times(1)).init((Configuration)any());
     verify(client.mockYarnClient, times(1)).start();
@@ -257,6 +260,7 @@ public class TestTezClient {
           (ShutdownSessionRequestProto) any());
     }
     verify(client.mockYarnClient, times(1)).stop();
+    verify(mockAcl, times(1)).close();
   }
   
   @Test (timeout=5000)

http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
index 3fa3db6..91ffe7b 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.hadoop.service.Service;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -32,7 +33,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.tez.common.security.ACLConfigurationParser;
 import org.apache.tez.common.security.ACLManager;
 import org.apache.tez.common.security.ACLType;
@@ -43,8 +43,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 
-import com.google.common.annotations.VisibleForTesting;
-
 public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager {
 
   private final static Logger LOG = LoggerFactory.getLogger(ATSHistoryACLPolicyManager.class);
@@ -250,4 +248,11 @@ public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager
{
     entity.setDomainId(domainId);
   }
 
+  @Override
+  public void close() {
+    if (timelineClient != null && timelineClient.isInState(Service.STATE.STARTED))
{
+      timelineClient.stop();
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
index eaf24d3..512913d 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.*;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -74,9 +75,6 @@ import com.sun.jersey.api.client.WebResource;
 
 import org.mockito.Matchers;
 
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.doThrow;
-
 public class TestATSHistoryWithACLs {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryWithACLs.class);
@@ -385,7 +383,9 @@ public class TestATSHistoryWithACLs {
     }
     dagLogging = dag2.getDagConf().get(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED);
     Assert.assertNull(dagLogging);
+    myAclPolicyManager.timelineClient = spy(myAclPolicyManager.timelineClient);
     tezSession.stop();
+    verify(myAclPolicyManager.timelineClient, times(1)).stop();
   }
   
 /**
@@ -465,7 +465,10 @@ public class TestATSHistoryWithACLs {
     }
     dagLogging = dag2.getDagConf().get(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED);
     Assert.assertNull(dagLogging);
+    myAclPolicyManager.timelineClient = spy(myAclPolicyManager.timelineClient);
     tezClient.stop();
+    verify(myAclPolicyManager.timelineClient, times(1)).stop();
+
   }
   /**
    * Test Disable Logging for all dags in a session 

http://git-wip-us.apache.org/repos/asf/tez/blob/9880c414/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index 6ea21e2..a66da24 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -243,6 +243,9 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
           + ", eventQueueBacklog=" + eventQueue.size());
     }
     timelineClient.stop();
+    if (historyACLPolicyManager != null) {
+      historyACLPolicyManager.close();
+    }
   }
 
   private void getEventBatch(List<DAGHistoryEvent> events) throws InterruptedException
{


Mime
View raw message