hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject hadoop git commit: YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne
Date Fri, 02 Nov 2018 00:10:31 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 a6d86c611 -> b561e0cfa


YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b561e0cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b561e0cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b561e0cf

Branch: refs/heads/branch-2.8
Commit: b561e0cfa3924586c5fc92ab3f5a358d39bbc9d9
Parents: a6d86c6
Author: Jonathan Hung <jhung@linkedin.com>
Authored: Thu Nov 1 14:21:08 2018 -0700
Committer: Jonathan Hung <jhung@linkedin.com>
Committed: Thu Nov 1 14:21:08 2018 -0700

----------------------------------------------------------------------
 .../server/resourcemanager/ClientRMService.java |  9 ++-
 .../server/resourcemanager/RMAuditLogger.java   | 66 +++++++++++++++++---
 .../scheduler/common/fica/FiCaSchedulerApp.java | 16 ++++-
 .../scheduler/fair/FSAppAttempt.java            |  4 +-
 .../resourcemanager/TestRMAuditLogger.java      | 22 ++++++-
 5 files changed, 100 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b561e0cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index c927a66..ea70643 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -580,7 +580,8 @@ public class ClientRMService extends AbstractService implements
       LOG.warn("Unable to get the current user.", ie);
       RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
           ie.getMessage(), "ClientRMService",
-          "Exception in submitting application", applicationId, callerContext);
+          "Exception in submitting application", applicationId, callerContext,
+          submissionContext.getQueue());
       throw RPCUtil.getRemoteException(ie);
     }
 
@@ -623,13 +624,15 @@ public class ClientRMService extends AbstractService implements
       LOG.info("Application with id " + applicationId.getId() + 
           " submitted by user " + user);
       RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST,
-          "ClientRMService", applicationId, callerContext);
+          "ClientRMService", applicationId, callerContext,
+          submissionContext.getQueue());
     } catch (YarnException e) {
       LOG.info("Exception in submitting application with id " +
           applicationId.getId(), e);
       RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
           e.getMessage(), "ClientRMService",
-          "Exception in submitting application", applicationId, callerContext);
+          "Exception in submitting application", applicationId, callerContext,
+          submissionContext.getQueue());
       throw e;
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b561e0cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index 07634c7..ee1a9fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -39,7 +39,7 @@ public class RMAuditLogger {
   static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
                     DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, 
                     CALLERCONTEXT, CALLERSIGNATURE, QUEUENAME,
-                    INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
+                    INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE, NODELABEL}
 
   public static class AuditConstants {
     static final String SUCCESS = "SUCCESS";
@@ -89,7 +89,7 @@ public class RMAuditLogger {
       ApplicationId appId, ApplicationAttemptId attemptId,
       ContainerId containerId) {
     return createSuccessLog(user, operation, target, appId, attemptId,
-        containerId, null, Server.getRemoteIp());
+        containerId, null, Server.getRemoteIp(), null, null);
   }
 
   /**
@@ -115,7 +115,7 @@ public class RMAuditLogger {
   static String createSuccessLog(String user, String operation, String target,
       ApplicationId appId, ApplicationAttemptId attemptId,
       ContainerId containerId, CallerContext callerContext,
-      InetAddress ip) {
+      InetAddress ip, String queueName, String partition) {
     StringBuilder b =
         createStringBuilderForSuccessEvent(user, operation, target, ip);
     if (appId != null) {
@@ -128,6 +128,12 @@ public class RMAuditLogger {
       add(Keys.CONTAINERID, containerId.toString(), b);
     }
     appendCallerContext(b, callerContext);
+    if (queueName != null) {
+      add(Keys.QUEUENAME, queueName, b);
+    }
+    if (partition != null) {
+      add(Keys.NODELABEL, partition, b);
+    }
     return b.toString();
   }
   
@@ -190,6 +196,30 @@ public class RMAuditLogger {
   }
 
   /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request to the ResourceManager
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed.
+   * @param appId Application Id in which operation was performed.
+   * @param containerId Container Id in which operation was performed.
+   * @param queueName Name of queue.
+   * @param partition Name of labeled partition.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, ContainerId containerId, String queueName,
+      String partition) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null,
+          containerId, null, null, queueName, partition));
+    }
+  }
+
+  /**
    * Create a general readable and parseable audit log string for a successful
    * event.
    *
@@ -255,7 +285,15 @@ public class RMAuditLogger {
       ApplicationId appId, CallerContext callerContext) {
     if (LOG.isInfoEnabled()) {
       LOG.info(createSuccessLog(user, operation, target, appId, null, null,
-          callerContext, Server.getRemoteIp()));
+          callerContext, Server.getRemoteIp(), null, null));
+    }
+  }
+
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, CallerContext callerContext, String queueName) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null, null,
+          callerContext, Server.getRemoteIp(), queueName, null));
     }
   }
 
@@ -283,7 +321,7 @@ public class RMAuditLogger {
       ApplicationId appId, InetAddress ip) {
     if (LOG.isInfoEnabled()) {
       LOG.info(createSuccessLog(user, operation, target, appId, null, null,
-          null, ip));
+          null, ip, null, null));
     }
   }
 
@@ -342,7 +380,7 @@ public class RMAuditLogger {
   static String createFailureLog(String user, String operation, String perm,
       String target, String description, ApplicationId appId,
       ApplicationAttemptId attemptId, ContainerId containerId,
-      CallerContext callerContext) {
+      CallerContext callerContext, String queueName) {
     StringBuilder b = createStringBuilderForFailureLog(user,
         operation, target, description, perm);
     if (appId != null) {
@@ -355,6 +393,9 @@ public class RMAuditLogger {
       add(Keys.CONTAINERID, containerId.toString(), b);
     }
     appendCallerContext(b, callerContext);
+    if (queueName != null) {
+      add(Keys.QUEUENAME, queueName, b);
+    }
     return b.toString();
   }
 
@@ -365,7 +406,7 @@ public class RMAuditLogger {
       String target, String description, ApplicationId appId,
       ApplicationAttemptId attemptId, ContainerId containerId) {
     return createFailureLog(user, operation, perm, target, description, appId,
-        attemptId, containerId, null);
+        attemptId, containerId, null, null);
   }
 
   /**
@@ -436,7 +477,16 @@ public class RMAuditLogger {
       CallerContext callerContext) {
     if (LOG.isWarnEnabled()) {
       LOG.warn(createFailureLog(user, operation, perm, target, description,
-          appId, null, null, callerContext));
+          appId, null, null, callerContext, null));
+    }
+  }
+
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      CallerContext callerContext, String queueName) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          appId, null, null, callerContext, queueName));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b561e0cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 782ad38..9663719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -166,9 +166,15 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
 
     containersToPreempt.remove(containerId);
 
+    // In order to save space in the audit log, only include the partition
+    // if it is not the default partition.
+    String containerPartition = null;
+    if (partition != null && !partition.isEmpty()) {
+      containerPartition = partition;
+    }
     RMAuditLogger.logSuccess(getUser(),
         AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
-        getApplicationId(), containerId);
+        getApplicationId(), containerId, getQueueName(), containerPartition);
     
     // Update usage metrics 
     Resource containerResource = rmContainer.getContainer().getResource();
@@ -230,9 +236,15 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
           + " container=" + containerId + " host="
           + container.getNodeId().getHost() + " type=" + type);
     }
+    // In order to save space in the audit log, only include the partition
+    // if it is not the default partition.
+    String partition = node.getPartition();
+    if (partition == null || partition.isEmpty()) {
+      partition = null;
+    }
     RMAuditLogger.logSuccess(getUser(),
         AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
-        getApplicationId(), containerId);
+        getApplicationId(), containerId, getQueueName(), partition);
     
     return rmContainer;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b561e0cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index fa57dbe..fb93285 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -150,7 +150,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
 
     RMAuditLogger.logSuccess(getUser(), 
         AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
-        getApplicationId(), containerId);
+        getApplicationId(), containerId, rmContainer.getQueueName(), null);
     
     // Update usage metrics 
     Resource containerResource = rmContainer.getContainer().getResource();
@@ -404,7 +404,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
     }
     RMAuditLogger.logSuccess(getUser(), 
         AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
-        getApplicationId(), container.getId());
+        getApplicationId(), container.getId(), getQueueName(), null);
     
     return rmContainer;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b561e0cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index 7ce7a49..c1d2967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -65,6 +65,7 @@ public class TestRMAuditLogger {
   private static final ContainerId CONTAINERID = mock(ContainerId.class);
   private static final String CALLER_CONTEXT = "context";
   private static final byte[] CALLER_SIGNATURE = "signature".getBytes();
+  private static final String PARTITION = "label1";
 
   @Before
   public void setUp() throws Exception {
@@ -128,6 +129,14 @@ public class TestRMAuditLogger {
       ApplicationAttemptId attemptId, ContainerId containerId,
       CallerContext callerContext, InetAddress remoteIp,
       RMAuditLogger.ArgsBuilder args) {
+    testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
+        callerContext, remoteIp, args, null, null);
+  }
+
+  private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+      ApplicationAttemptId attemptId, ContainerId containerId,
+      CallerContext callerContext, InetAddress remoteIp,
+      RMAuditLogger.ArgsBuilder args, String queueName, String partition) {
     String sLog;
     InetAddress tmpIp = checkIP ? remoteIp : null;
     if (args != null) {
@@ -135,7 +144,7 @@ public class TestRMAuditLogger {
           tmpIp, args);
     } else {
       sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
-          attemptId, containerId, callerContext, tmpIp);
+          attemptId, containerId, callerContext, tmpIp, queueName, partition);
     }
     StringBuilder expLog = new StringBuilder();
     expLog.append("USER=test\t");
@@ -170,6 +179,13 @@ public class TestRMAuditLogger {
     if (args != null) {
       expLog.append("\tQUEUENAME=root");
       expLog.append("\tRECURSIVE=true");
+    } else {
+      if (queueName != null) {
+        expLog.append("\tQUEUENAME=" + QUEUE);
+      }
+    }
+    if (partition != null) {
+      expLog.append("\tNODELABEL=" + PARTITION);
     }
     assertEquals(expLog.toString(), sLog);
   }
@@ -251,6 +267,8 @@ public class TestRMAuditLogger {
         .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
     testSuccessLogFormatHelper(checkIP, null, null, null, null,
         Server.getRemoteIp(), args);
+    testSuccessLogFormatHelper(checkIP, null, null, null, null,
+        Server.getRemoteIp(), null, QUEUE, PARTITION);
     testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
     testSuccessLogNulls(checkIP);
   }
@@ -275,7 +293,7 @@ public class TestRMAuditLogger {
         CallerContext callerContext, RMAuditLogger.ArgsBuilder args) {
     String fLog = args == null ?
       RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
-          appId, attemptId, containerId, callerContext) :
+          appId, attemptId, containerId, callerContext, null) :
         RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
             args);
     StringBuilder expLog = new StringBuilder();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message