From common-commits-return-90401-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Nov 6 19:05:42 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 2146918067E for ; Tue, 6 Nov 2018 19:05:40 +0100 (CET) Received: (qmail 62200 invoked by uid 500); 6 Nov 2018 18:05:37 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 61604 invoked by uid 99); 6 Nov 2018 18:05:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Nov 2018 18:05:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CFE98E1219; Tue, 6 Nov 2018 18:05:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: brahma@apache.org To: common-commits@hadoop.apache.org Date: Tue, 06 Nov 2018 18:05:52 -0000 Message-Id: <0482ca4c89ff478996d8b2ce3e968e84@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [18/48] hadoop git commit: YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ab611d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ab611d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ab611d4 Branch: refs/heads/HDFS-13532 Commit: 2ab611d48b7669b31bd2c9b918f47251da77d0f6 Parents: d174b91 Author: Jonathan Hung Authored: Thu Nov 1 14:22:00 2018 -0700 Committer: Jonathan Hung Committed: Thu Nov 1 14:22:00 2018 -0700 ---------------------------------------------------------------------- .../server/resourcemanager/ClientRMService.java | 12 ++- .../server/resourcemanager/RMAuditLogger.java | 81 +++++++++++++++++--- .../scheduler/common/fica/FiCaSchedulerApp.java | 19 ++++- .../scheduler/fair/FSAppAttempt.java | 5 +- .../scheduler/fifo/FifoAppAttempt.java | 10 ++- .../resourcemanager/TestRMAuditLogger.java | 23 +++++- 6 files changed, 129 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 8f8f43e..2da000c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -580,7 +580,8 @@ public class ClientRMService extends AbstractService implements LOG.warn("Unable to get the current user.", ie); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, ie.getMessage(), "ClientRMService", - "Exception in submitting application", applicationId, callerContext); + "Exception in submitting application", applicationId, callerContext, + submissionContext.getQueue()); throw RPCUtil.getRemoteException(ie); } @@ -603,7 +604,8 @@ public class ClientRMService extends AbstractService implements ". Flow run should be a long integer", e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", - "Exception in submitting application", applicationId); + "Exception in submitting application", applicationId, + submissionContext.getQueue()); throw RPCUtil.getRemoteException(e); } } @@ -662,12 +664,14 @@ public class ClientRMService extends AbstractService implements LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, - "ClientRMService", applicationId, callerContext); + "ClientRMService", applicationId, callerContext, + submissionContext.getQueue()); } catch (YarnException e) { LOG.info("Exception in submitting " + applicationId, e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", - "Exception in submitting application", applicationId, callerContext); + "Exception in submitting application", applicationId, callerContext, + submissionContext.getQueue()); throw e; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index ab10895..292aa8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -40,7 +40,7 @@ public class RMAuditLogger { enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME, - INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE} + INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE, NODELABEL} public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -98,7 +98,7 @@ public class RMAuditLogger { ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) { return createSuccessLog(user, operation, target, appId, attemptId, - containerId, resource, null, Server.getRemoteIp()); + containerId, resource, null, Server.getRemoteIp(), null, null); } /** @@ -124,7 +124,7 @@ public class RMAuditLogger { static String createSuccessLog(String user, String operation, String target, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, Resource resource, CallerContext callerContext, - InetAddress ip) { + InetAddress ip, String queueName, String partition) { StringBuilder b = createStringBuilderForSuccessEvent(user, operation, target, ip); if (appId != null) { @@ -140,6 +140,12 @@ public class RMAuditLogger { add(Keys.RESOURCE, resource.toString(), b); } appendCallerContext(b, callerContext); + if (queueName != null) { + add(Keys.QUEUENAME, queueName, b); + } + if (partition != null) { + add(Keys.NODELABEL, partition, b); + } return b.toString(); } @@ -203,6 +209,32 @@ public class RMAuditLogger { } /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param containerId Container Id in which operation was performed. + * @param resource Resource associated with container. + * @param queueName Name of queue. + * @param partition Name of labeled partition. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, ContainerId containerId, Resource resource, + String queueName, String partition) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null, + containerId, resource, null, Server.getRemoteIp(), queueName, + partition)); + } + } + + /** * Create a general readable and parseable audit log string for a successful * event. * @@ -263,12 +295,20 @@ public class RMAuditLogger { null, null)); } } - + public static void logSuccess(String user, String operation, String target, ApplicationId appId, CallerContext callerContext) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, null, null, - null, callerContext, Server.getRemoteIp())); + null, callerContext, Server.getRemoteIp(), null, null)); + } + } + + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, CallerContext callerContext, String queueName) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null, null, + null, callerContext, Server.getRemoteIp(), queueName, null)); } } @@ -296,7 +336,7 @@ public class RMAuditLogger { ApplicationId appId, InetAddress ip) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, null, null, - null, null, ip)); + null, null, ip, null, null)); } } @@ -355,7 +395,7 @@ public class RMAuditLogger { static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - Resource resource, CallerContext callerContext) { + Resource resource, CallerContext callerContext, String queueName) { StringBuilder b = createStringBuilderForFailureLog(user, operation, target, description, perm); if (appId != null) { @@ -371,6 +411,9 @@ public class RMAuditLogger { add(Keys.RESOURCE, resource.toString(), b); } appendCallerContext(b, callerContext); + if (queueName != null) { + add(Keys.QUEUENAME, queueName, b); + } return b.toString(); } @@ -381,7 +424,7 @@ public class RMAuditLogger { String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) { return createFailureLog(user, operation, perm, target, description, appId, - attemptId, containerId, resource, null); + attemptId, containerId, resource, null, null); } /** @@ -447,13 +490,22 @@ public class RMAuditLogger { appId, attemptId, null, null)); } } - + public static void logFailure(String user, String operation, String perm, String target, String description, ApplicationId appId, CallerContext callerContext) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, callerContext)); + appId, null, null, null, callerContext, null)); + } + } + + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + CallerContext callerContext, String queueName) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + appId, null, null, null, callerContext, queueName)); } } @@ -480,6 +532,15 @@ public class RMAuditLogger { } } + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + String queueName) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + appId, null, null, null, null, queueName)); + } + } + /** * Create a readable and parseable audit log string for a failed event. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 4bfdae9..114ee51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -218,9 +218,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { containersToPreempt.remove(containerId); + // In order to save space in the audit log, only include the partition + // if it is not the default partition. + String containerPartition = null; + if (partition != null && !partition.isEmpty()) { + containerPartition = partition; + } Resource containerResource = rmContainer.getContainer().getResource(); RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, - "SchedulerApp", getApplicationId(), containerId, containerResource); + "SchedulerApp", getApplicationId(), containerId, containerResource, + getQueueName(), containerPartition); // Update usage metrics queue.getMetrics().releaseResources(partition, @@ -646,9 +653,17 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { + " host=" + rmContainer.getAllocatedNode().getHost() + " type=" + allocation.getAllocationLocalityType()); } + // In order to save space in the audit log, only include the partition + // if it is not the default partition. + String partition = + schedulerContainer.getSchedulerNode().getPartition(); + if (partition != null && partition.isEmpty()) { + partition = null; + } RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, - allocation.getAllocatedOrReservedResource()); + allocation.getAllocatedOrReservedResource(), getQueueName(), + partition); } else { // If the rmContainer's state is already updated to RESERVED, this is // a reReservation http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index d9f3262..2a6657a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -162,7 +162,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt Resource containerResource = rmContainer.getContainer().getResource(); RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, - "SchedulerApp", getApplicationId(), containerId, containerResource); + "SchedulerApp", getApplicationId(), containerId, containerResource, + rmContainer.getQueueName(), null); // Update usage metrics queue.getMetrics().releaseResources( @@ -479,7 +480,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), container.getId(), - container.getResource()); + container.getResource(), getQueueName(), null); } finally { writeLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java index 169b98a..074d5da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java @@ -99,9 +99,17 @@ public class FifoAppAttempt extends FiCaSchedulerApp { .getApplicationAttemptId() + " container=" + containerId + " host=" + container.getNodeId().getHost() + " type=" + type); } + // In order to save space in the audit log, only include the partition + // if it is not the default partition. + String partition = null; + if (appAMNodePartitionName != null && + !appAMNodePartitionName.isEmpty()) { + partition = appAMNodePartitionName; + } RMAuditLogger.logSuccess(getUser(), RMAuditLogger.AuditConstants.ALLOC_CONTAINER, "SchedulerApp", - getApplicationId(), containerId, container.getResource()); + getApplicationId(), containerId, container.getResource(), + getQueueName(), partition); return rmContainer; } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index d5f1e35..7d8eeab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -67,6 +67,7 @@ public class TestRMAuditLogger { private static final Resource RESOURCE = mock(Resource.class); private static final String CALLER_CONTEXT = "context"; private static final byte[] CALLER_SIGNATURE = "signature".getBytes(); + private static final String PARTITION = "label1"; @Before public void setUp() throws Exception { @@ -132,6 +133,14 @@ public class TestRMAuditLogger { ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource, InetAddress remoteIp, RMAuditLogger.ArgsBuilder args) { + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, + callerContext, resource, remoteIp, args, null, null); + } + + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext, Resource resource, InetAddress remoteIp, + RMAuditLogger.ArgsBuilder args, String queueName, String partition) { String sLog; InetAddress tmpIp = checkIP ? remoteIp : null; if (args != null) { @@ -139,7 +148,8 @@ public class TestRMAuditLogger { tmpIp, args); } else { sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, resource, callerContext, tmpIp); + attemptId, containerId, resource, callerContext, tmpIp, queueName, + partition); } StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); @@ -177,6 +187,13 @@ public class TestRMAuditLogger { if (args != null) { expLog.append("\tQUEUENAME=root"); expLog.append("\tRECURSIVE=true"); + } else { + if (queueName != null) { + expLog.append("\tQUEUENAME=" + QUEUE); + } + } + if (partition != null) { + expLog.append("\tNODELABEL=" + PARTITION); } assertEquals(expLog.toString(), sLog); } @@ -258,6 +275,8 @@ public class TestRMAuditLogger { .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); testSuccessLogFormatHelper(checkIP, null, null, null, null, null, Server.getRemoteIp(), args); + testSuccessLogFormatHelper(checkIP, null, null, null, null, null, + Server.getRemoteIp(), null, QUEUE, PARTITION); testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID); testSuccessLogNulls(checkIP); } @@ -283,7 +302,7 @@ public class TestRMAuditLogger { RMAuditLogger.ArgsBuilder args) { String fLog = args == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, resource, callerContext) : + appId, attemptId, containerId, resource, callerContext, null) : RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, args); StringBuilder expLog = new StringBuilder(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org