hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1165950 - in /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server: hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ hadoop-yarn-server-nodemanager/src/test/java/org/apache/hado...
Date Wed, 07 Sep 2011 01:26:21 GMT
Author: acmurthy
Date: Wed Sep  7 01:26:21 2011
New Revision: 1165950

URL: http://svn.apache.org/viewvc?rev=1165950&view=rev
Log:
Adding new files for MAPREDUCE-2655.

Added:
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java

Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
(added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
Wed Sep  7 01:26:21 2011
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.net.InetAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/** 
+ * Manages NodeManager audit logs.
+ *
+ * Audit log format is written as key=value pairs. Tab separated.
+ */
+public class NMAuditLogger {
+  private static final Log LOG = LogFactory.getLog(NMAuditLogger.class);
+
+  static enum Keys {USER, OPERATION, TARGET, RESULT, IP, 
+                    DESCRIPTION, APPID, CONTAINERID}
+
+  public static class AuditConstants {
+    static final String SUCCESS = "SUCCESS";
+    static final String FAILURE = "FAILURE";
+    static final String KEY_VAL_SEPARATOR = "=";
+    static final char PAIR_SEPARATOR = '\t';
+
+    // Some commonly used descriptions
+    public static final String START_CONTAINER = "Start Container Request";
+    public static final String STOP_CONTAINER = "Stop Container Request";
+    public static final String FINISH_SUCCESS_CONTAINER = "Container Finished - Succeeded";
+    public static final String FINISH_FAILED_CONTAINER = "Container Finished - Failed";
+    public static final String FINISH_KILLED_CONTAINER = "Container Finished - Killed";
+  }
+
+  /**
+   * A helper api for creating an audit log for a successful event.
+   */
+  static String createSuccessLog(String user, String operation, String target, 
+      ApplicationId appId, ContainerId containerId) {
+    StringBuilder b = new StringBuilder();
+    start(Keys.USER, user, b);
+    addRemoteIP(b);
+    add(Keys.OPERATION, operation, b);
+    add(Keys.TARGET, target ,b);
+    add(Keys.RESULT, AuditConstants.SUCCESS, b);
+    if (appId != null) {
+      add(Keys.APPID, appId.toString(), b);
+    }
+    if (containerId != null) {
+      add(Keys.CONTAINERID, containerId.toString(), b);
+    }
+    return b.toString();
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user
+   * @param target The target on which the operation is being performed.
+   * @param appId Application Id in which operation was performed.
+   * @param containerId Container Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, ContainerId containerId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, containerId));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user
+   * @param target The target on which the operation is being performed.
+   *
+   * <br><br>
+   * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, null, null));
+    }
+  }
+
+  /**
+   * A helper api for creating an audit log for a failure event.
+   * This is factored out for testing purpose.
+   */
+  static String createFailureLog(String user, String operation, String target, 
+      String description, ApplicationId appId, ContainerId containerId) {
+    StringBuilder b = new StringBuilder();
+    start(Keys.USER, user, b);
+    addRemoteIP(b);
+    add(Keys.OPERATION, operation, b);
+    add(Keys.TARGET, target ,b);
+    add(Keys.RESULT, AuditConstants.FAILURE, b);
+    add(Keys.DESCRIPTION, description, b);
+    if (appId != null) {
+      add(Keys.APPID, appId.toString(), b);
+    }
+    if (containerId != null) {
+      add(Keys.CONTAINERID, containerId.toString(), b);
+    }
+    return b.toString();
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed. 
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   * @param appId ApplicationId in which operation was performed.
+   * @param containerId Container Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String target, 
+      String description, ApplicationId appId, ContainerId containerId) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, target, description, appId, containerId));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed. 
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   *
+   * <br><br>
+   * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, 
+                         String target, String description) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, target, description, null, null));
+    }
+  }
+
+  /**
+   * A helper api to add remote IP address
+   */
+  static void addRemoteIP(StringBuilder b) {
+    InetAddress ip = Server.getRemoteIp();
+    // ip address can be null for testcases
+    if (ip != null) {
+      add(Keys.IP, ip.getHostAddress(), b);
+    }
+  }
+
+  /**
+   * Adds the first key-val pair to the passed builder in the following format
+   * key=value
+   */
+  static void start(Keys key, String value, StringBuilder b) {
+    b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+  }
+
+  /**
+   * Appends the key-val pair to the passed builder in the following format
+   * <pair-delim>key=value
+   */
+  static void add(Keys key, String value, StringBuilder b) {
+    b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
+     .append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+  }
+}

Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
(added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
Wed Sep  7 01:26:21 2011
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.TestRPC.TestImpl;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
+
+import org.apache.hadoop.net.NetUtils;
+
+import static org.mockito.Mockito.*;
+import static junit.framework.Assert.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Tests {@link NMAuditLogger}.
+ */
+public class TestNMAuditLogger {
+  private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class);
+  private static final String USER = "test";
+  private static final String OPERATION = "oper";
+  private static final String TARGET = "tgt";
+  private static final String DESC = "description of an audit log";
+
+  private static final ApplicationId APPID = mock(ApplicationId.class);
+  private static final ContainerId CONTAINERID = mock(ContainerId.class);
+
+  @Before
+  public void setUp() throws Exception {
+    when(APPID.toString()).thenReturn("app_1");
+    when(CONTAINERID.toString()).thenReturn("container_1");
+  }
+
+
+  /**
+   * Test the AuditLog format with key-val pair.
+   */
+  @Test  
+  public void testKeyValLogFormat() throws Exception {
+    StringBuilder actLog = new StringBuilder();
+    StringBuilder expLog = new StringBuilder();
+    // add the first k=v pair and check
+    NMAuditLogger.start(Keys.USER, USER, actLog);
+    expLog.append("USER=test");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // append another k1=v1 pair to already added k=v and test
+    NMAuditLogger.add(Keys.OPERATION, OPERATION, actLog);
+    expLog.append("\tOPERATION=oper");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // append another k1=null pair and test
+    NMAuditLogger.add(Keys.APPID, (String)null, actLog);
+    expLog.append("\tAPPID=null");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // now add the target and check of the final string
+    NMAuditLogger.add(Keys.TARGET, TARGET, actLog);
+    expLog.append("\tTARGET=tgt");
+    assertEquals(expLog.toString(), actLog.toString());
+  }
+
+
+  /**
+   * Test the AuditLog format for successful events.
+   */
+  private void testSuccessLogFormatHelper(boolean checkIP, 
+      ApplicationId appId, ContainerId containerId) {
+    // check without the IP
+    String sLog = NMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+        appId, containerId);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=test\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
+    if (appId != null) {
+      expLog.append("\tAPPID=app_1");
+    }
+    if (containerId != null) {
+      expLog.append("\tCONTAINERID=container_1");
+    }
+    assertEquals(expLog.toString(), sLog);
+  }
+
+  /**
+   * Test the AuditLog format for successful events passing nulls.
+   */
+  private void testSuccessLogNulls(boolean checkIP) {
+    String sLog = NMAuditLogger.createSuccessLog(null, null, null,
+        null, null);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=null\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
+    assertEquals(expLog.toString(), sLog);
+  }
+
+  /**
+   * Test the AuditLog format for successful events with the various
+   * parameters.
+   */
+  private void testSuccessLogFormat(boolean checkIP) {
+    testSuccessLogFormatHelper(checkIP, null, null);
+    testSuccessLogFormatHelper(checkIP, APPID, null);
+    testSuccessLogFormatHelper(checkIP, null, CONTAINERID);
+    testSuccessLogFormatHelper(checkIP, APPID, CONTAINERID);
+    testSuccessLogNulls(checkIP);
+  }
+
+
+  /**
+   * Test the AuditLog format for failure events.
+   */
+  private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+      ContainerId containerId) {
+    String fLog =
+      NMAuditLogger.createFailureLog(USER, OPERATION, TARGET, DESC, appId,
+      containerId);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=test\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
+    expLog.append("DESCRIPTION=description of an audit log");
+
+    if (appId != null) {
+      expLog.append("\tAPPID=app_1");
+    }
+    if (containerId != null) {
+      expLog.append("\tCONTAINERID=container_1");
+    }
+    assertEquals(expLog.toString(), fLog);
+  }
+
+  /**
+   * Test the AuditLog format for failure events with the various
+   * parameters.
+   */
+  private void testFailureLogFormat(boolean checkIP) {
+    testFailureLogFormatHelper(checkIP, null, null);
+    testFailureLogFormatHelper(checkIP, APPID, null);
+    testFailureLogFormatHelper(checkIP, null, CONTAINERID);
+    testFailureLogFormatHelper(checkIP, APPID, CONTAINERID);
+  }
+
+  /**
+   * Test {@link NMAuditLogger} without IP set.
+   */
+  @Test  
+  public void testNMAuditLoggerWithoutIP() throws Exception {
+    // test without ip
+    testSuccessLogFormat(false);
+    testFailureLogFormat(false);
+  }
+
+  /**
+   * A special extension of {@link TestImpl} RPC server with 
+   * {@link TestImpl#ping()} testing the audit logs.
+   */
+  private class MyTestRPCServer extends TestImpl {
+    @Override
+    public void ping() {
+      // test with ip set
+      testSuccessLogFormat(true);
+      testFailureLogFormat(true);
+    }
+  }
+
+  /**
+   * Test {@link NMAuditLogger} with IP set.
+   */
+  @Test  
+  public void testNMAuditLoggerWithIP() throws Exception {
+    Configuration conf = new Configuration();
+    // start the IPC server
+    Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
+    server.start();
+
+    InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
+    // Make a client connection and test the audit log
+    TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
+                           TestProtocol.versionID, addr, conf);
+    // Start the testcase
+    proxy.ping();
+
+    server.stop();
+  }
+}

Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
(added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
Wed Sep  7 01:26:21 2011
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.net.InetAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/** 
+ * Manages ResourceManager audit logs. 
+ *
+ * Audit log format is written as key=value pairs. Tab separated.
+ */
+public class RMAuditLogger {
+  private static final Log LOG = LogFactory.getLog(RMAuditLogger.class);
+
+  static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
+                    DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID}
+
+  public static class AuditConstants {
+    static final String SUCCESS = "SUCCESS";
+    static final String FAILURE = "FAILURE";
+    static final String KEY_VAL_SEPARATOR = "=";
+    static final char PAIR_SEPARATOR = '\t';
+
+    public static final String KILL_APP_REQUEST = "Kill Application Request";
+    public static final String SUBMIT_APP_REQUEST = "Submit Application Request";
+    public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
+    public static final String FINISH_FAILED_APP = "Application Finished - Failed";
+    public static final String FINISH_KILLED_APP = "Application Finished - Killed";
+    public static final String REGISTER_AM = "Register App Master";
+    public static final String ALLOC_CONTAINER = "AM Allocated Container";
+    public static final String RELEASE_CONTAINER = "AM Released Container";
+
+    // Some commonly used descriptions
+    public static final String UNAUTHORIZED_USER = "Unauthorized user";
+  }
+
+  /**
+   * A helper api for creating an audit log for a successful event.
+   */
+  static String createSuccessLog(String user, String operation, String target,
+      ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) {
+    StringBuilder b = new StringBuilder();
+    start(Keys.USER, user, b);
+    addRemoteIP(b);
+    add(Keys.OPERATION, operation, b);
+    add(Keys.TARGET, target ,b);
+    add(Keys.RESULT, AuditConstants.SUCCESS, b);
+    if (appId != null) {
+      add(Keys.APPID, appId.toString(), b);
+    }
+    if (attemptId != null) {
+      add(Keys.APPATTEMPTID, attemptId.toString(), b);
+    }
+    if (containerId != null) {
+      add(Keys.CONTAINERID, containerId.toString(), b);
+    }
+    return b.toString();
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request to the ResourceManager
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed. 
+   * @param appId Application Id in which operation was performed.
+   * @param containerId Container Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target, 
+      ApplicationId appId, ContainerId containerId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null, 
+          containerId));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request to the ResourceManager.
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed. 
+   * @param appId Application Id in which operation was performed.
+   * @param attemptId Application Attempt Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target, 
+      ApplicationId appId, ApplicationAttemptId attemptId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, attemptId,
+          null));
+    }
+  }
+
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request to the ResourceManager.
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed. 
+   * @param appId Application Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null, null));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed. 
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, null, null, null));
+    }
+  }
+
+  /**
+   * A helper api for creating an audit log for a failure event.
+   */
+  static String createFailureLog(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      ApplicationAttemptId attemptId, ContainerId containerId) {
+    StringBuilder b = new StringBuilder();
+    start(Keys.USER, user, b);
+    addRemoteIP(b);
+    add(Keys.OPERATION, operation, b);
+    add(Keys.TARGET, target ,b);
+    add(Keys.RESULT, AuditConstants.FAILURE, b);
+    add(Keys.DESCRIPTION, description, b);
+    add(Keys.PERMISSIONS, perm, b);
+    if (appId != null) {
+      add(Keys.APPID, appId.toString(), b);
+    }
+    if (attemptId != null) {
+      add(Keys.APPATTEMPTID, attemptId.toString(), b);
+    }
+    if (containerId != null) {
+      add(Keys.CONTAINERID, containerId.toString(), b);
+    }
+    return b.toString();
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions. 
+   * @param target The target on which the operation is being performed. 
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   * @param appId Application Id in which operation was performed.
+   * @param containerId Container Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId, 
+      ContainerId containerId) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          appId, null, containerId));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed. 
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   * @param appId ApplicationId in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId, 
+      ApplicationAttemptId attemptId) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          appId, attemptId, null));
+    }
+  }
+
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request. 
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed. 
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   * @param appId ApplicationId in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          appId, null, null));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request.
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions. 
+   * @param target The target on which the operation is being performed. 
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          null, null, null));
+    }
+  }
+
+  /**
+   * A helper api to add remote IP address
+   */
+  static void addRemoteIP(StringBuilder b) {
+    InetAddress ip = Server.getRemoteIp();
+    // ip address can be null for testcases
+    if (ip != null) {
+      add(Keys.IP, ip.getHostAddress(), b);
+    }
+  }
+
+  /**
+   * Adds the first key-val pair to the passed builder in the following format
+   * key=value
+   */
+  static void start(Keys key, String value, StringBuilder b) {
+    b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+  }
+
+  /**
+   * Appends the key-val pair to the passed builder in the following format
+   * <pair-delim>key=value
+   */
+  static void add(Keys key, String value, StringBuilder b) {
+    b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
+     .append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+  }
+}

Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java?rev=1165950&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
(added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
Wed Sep  7 01:26:21 2011
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.TestRPC.TestImpl;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
+
+import org.apache.hadoop.net.NetUtils;
+
+import static org.mockito.Mockito.*;
+import static junit.framework.Assert.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Tests {@link RMAuditLogger}.
+ */
+public class TestRMAuditLogger {
+  private static final Log LOG = LogFactory.getLog(TestRMAuditLogger.class);
+  private static final String USER = "test";
+  private static final String OPERATION = "oper";
+  private static final String TARGET = "tgt";
+  private static final String PERM = "admin group";
+  private static final String DESC = "description of an audit log";
+  private static final ApplicationId APPID = mock(ApplicationId.class);
+  private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
+  private static final ContainerId CONTAINERID = mock(ContainerId.class);
+
+  @Before
+  public void setUp() throws Exception {
+    when(APPID.toString()).thenReturn("app_1");
+    when(ATTEMPTID.toString()).thenReturn("app_attempt_1");
+    when(CONTAINERID.toString()).thenReturn("container_1");
+  }
+
+
+  /**
+   * Test the AuditLog format with key-val pair.
+   */
+  @Test  
+  public void testKeyValLogFormat() throws Exception {
+    StringBuilder actLog = new StringBuilder();
+    StringBuilder expLog = new StringBuilder();
+    // add the first k=v pair and check
+    RMAuditLogger.start(Keys.USER, USER, actLog);
+    expLog.append("USER=test");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // append another k1=v1 pair to already added k=v and test
+    RMAuditLogger.add(Keys.OPERATION, OPERATION, actLog);
+    expLog.append("\tOPERATION=oper");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // append another k1=null pair and test
+    RMAuditLogger.add(Keys.APPID, (String)null, actLog);
+    expLog.append("\tAPPID=null");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // now add the target and check of the final string
+    RMAuditLogger.add(Keys.TARGET, TARGET, actLog);
+    expLog.append("\tTARGET=tgt");
+    assertEquals(expLog.toString(), actLog.toString());
+  }
+
+
+  /**
+   * Test the AuditLog format for successful events.
+   */
+  private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+      ApplicationAttemptId attemptId, ContainerId containerId) {
+    String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+        appId, attemptId, containerId);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=test\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
+
+    if (appId != null) {
+      expLog.append("\tAPPID=app_1");
+    }
+    if (attemptId != null) {
+      expLog.append("\tAPPATTEMPTID=app_attempt_1");
+    }
+    if (containerId != null) {
+      expLog.append("\tCONTAINERID=container_1");
+    }
+    assertEquals(expLog.toString(), sLog);
+  }
+
+  /**
+   * Test the AuditLog format for successful events passing nulls.
+   */
+  private void testSuccessLogNulls(boolean checkIP) {
+    String sLog = RMAuditLogger.createSuccessLog(null, null, null, null, 
+        null, null);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=null\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
+    assertEquals(expLog.toString(), sLog);
+  }
+
+  /**
+   * Test the AuditLog format for successful events with the various
+   * parameters.
+   */
+  private void testSuccessLogFormat(boolean checkIP) {
+    testSuccessLogFormatHelper(checkIP, null, null, null);
+    testSuccessLogFormatHelper(checkIP, APPID, null, null);
+    testSuccessLogFormatHelper(checkIP, null, null, CONTAINERID);
+    testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, null);
+    testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, null);
+    testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID);
+    testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
+    testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
+    testSuccessLogNulls(checkIP);
+  }
+
+
+  /**
+   * Test the AuditLog format for failure events.
+   */
+  private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+      ApplicationAttemptId attemptId, ContainerId containerId) {
+    String fLog =
+      RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
+      appId, attemptId, containerId);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=test\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
+    expLog.append("DESCRIPTION=description of an audit log");
+    expLog.append("\tPERMISSIONS=admin group");
+    if (appId != null) {
+      expLog.append("\tAPPID=app_1");
+    }
+    if (attemptId != null) {
+      expLog.append("\tAPPATTEMPTID=app_attempt_1");
+    }
+    if (containerId != null) {
+      expLog.append("\tCONTAINERID=container_1");
+    }
+    assertEquals(expLog.toString(), fLog);
+  }
+
+  /**
+   * Test the AuditLog format for failure events with the various
+   * parameters.
+   */
+  private void testFailureLogFormat(boolean checkIP) {
+    testFailureLogFormatHelper(checkIP, null, null, null);
+    testFailureLogFormatHelper(checkIP, APPID, null, null);
+    testFailureLogFormatHelper(checkIP, null, null, CONTAINERID);
+    testFailureLogFormatHelper(checkIP, null, ATTEMPTID, null);
+    testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, null);
+    testFailureLogFormatHelper(checkIP, APPID, null, CONTAINERID);
+    testFailureLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
+    testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
+  }
+
+  /**
+   * Test {@link RMAuditLogger} without IP set.
+   */
+  @Test  
+  public void testRMAuditLoggerWithoutIP() throws Exception {
+    // test without ip
+    testSuccessLogFormat(false);
+    testFailureLogFormat(false);
+  }
+
+  /**
+   * A special extension of {@link TestImpl} RPC server with 
+   * {@link TestImpl#ping()} testing the audit logs.
+   */
+  private class MyTestRPCServer extends TestImpl {
+    @Override
+    public void ping() {
+      // test with ip set
+      testSuccessLogFormat(true);
+      testFailureLogFormat(true);
+    }
+  }
+
+  /**
+   * Test {@link RMAuditLogger} with IP set.
+   */
+  @Test  
+  public void testRMAuditLoggerWithIP() throws Exception {
+    Configuration conf = new Configuration();
+    // start the IPC server
+    Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
+    server.start();
+
+    InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
+    // Make a client connection and test the audit log
+    TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
+                           TestProtocol.versionID, addr, conf);
+    // Start the testcase
+    proxy.ping();
+
+    server.stop();
+  }
+}



Mime
View raw message