Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 54ED518EA6 for ; Fri, 15 May 2015 05:36:59 +0000 (UTC) Received: (qmail 81339 invoked by uid 500); 15 May 2015 05:36:59 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 81276 invoked by uid 500); 15 May 2015 05:36:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 81267 invoked by uid 99); 15 May 2015 05:36:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 May 2015 05:36:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EB90FE0978; Fri, 15 May 2015 05:36:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Message-Id: <12039301e050463eb05d38e7a80d6ee9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-6888. Allow selectively audit logging ops (Contributed by Chen He) Date: Fri, 15 May 2015 05:36:58 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 91855c234 -> d9455c790 HDFS-6888. Allow selectively audit logging ops (Contributed by Chen He) (cherry picked from commit 7f2e89fa7082840bfa3e8e593c93db050a80d04f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9455c79 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9455c79 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9455c79 Branch: refs/heads/branch-2 Commit: d9455c790f6a7539ba411280bd836945977e39ab Parents: 91855c2 Author: Vinayakumar B Authored: Fri May 15 11:05:01 2015 +0530 Committer: Vinayakumar B Committed: Fri May 15 11:05:26 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 + .../hdfs/server/namenode/FSNamesystem.java | 11 +- .../src/main/resources/hdfs-default.xml | 9 ++ .../server/namenode/TestAuditLogAtDebug.java | 131 +++++++++++++++++++ 5 files changed, 152 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9455c79/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3105bb4..62b1d97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -219,6 +219,8 @@ Release 2.8.0 - UNRELEASED HDFS-8350. Remove old webhdfs.xml and other outdated documentation stuff. (Brahma Reddy Battula via aajisaka) + HDFS-6888. Allow selectively audit logging ops (Chen He via vinayakumarb) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9455c79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 7d2a25b..3669685 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -339,6 +339,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT = false; public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY = "dfs.namenode.audit.log.async"; public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT = false; + public static final String DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST = "dfs.namenode.audit.log.debug.cmdlist"; public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth"; public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9455c79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index bf92c71..8fe32fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -8143,15 +8143,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, * defined in the config file. It can also be explicitly listed in the * config file. */ - private static class DefaultAuditLogger extends HdfsAuditLogger { + @VisibleForTesting + static class DefaultAuditLogger extends HdfsAuditLogger { private boolean logTokenTrackingId; + private Set debugCmdSet = new HashSet(); @Override public void initialize(Configuration conf) { logTokenTrackingId = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_KEY, DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT); + + debugCmdSet.addAll(Arrays.asList(conf.getTrimmedStrings( + DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST))); } @Override @@ -8159,7 +8164,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, InetAddress addr, String cmd, String src, String dst, FileStatus status, UserGroupInformation ugi, DelegationTokenSecretManager dtSecretManager) { - if (auditLog.isInfoEnabled()) { + + if (auditLog.isDebugEnabled() || + (auditLog.isInfoEnabled() && !debugCmdSet.contains(cmd))) { final StringBuilder sb = auditBuffer.get(); sb.setLength(0); sb.append("allowed=").append(succeeded).append("\t"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9455c79/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5396f82..d0dd49d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2084,6 +2084,15 @@ + dfs.namenode.audit.log.debug.cmdlist + + + A comma separated list of NameNode commands that are written to the HDFS + namenode audit log only if the audit log level is debug. + + + + dfs.client.use.legacy.blockreader.local false http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9455c79/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java new file mode 100644 index 0000000..ce11514 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.DefaultAuditLogger; +import org.apache.log4j.Level; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.net.Inet4Address; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.*; + +/** + * Test that the HDFS Audit logger respects DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST. + */ +public class TestAuditLogAtDebug { + static final Log LOG = LogFactory.getLog(TestAuditLogAtDebug.class); + + @Rule + public Timeout timeout = new Timeout(300000); + + private static final String DUMMY_COMMAND_1 = "dummycommand1"; + private static final String DUMMY_COMMAND_2 = "dummycommand2"; + + private DefaultAuditLogger makeSpyLogger( + Level level, Optional> debugCommands) { + DefaultAuditLogger logger = new DefaultAuditLogger(); + Configuration conf = new HdfsConfiguration(); + if (debugCommands.isPresent()) { + conf.set(DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST, + Joiner.on(",").join(debugCommands.get())); + } + logger.initialize(conf); + ((Log4JLogger) FSNamesystem.auditLog).getLogger().setLevel(level); + return spy(logger); + } + + private void logDummyCommandToAuditLog(HdfsAuditLogger logger, String command) { + logger.logAuditEvent(true, "", + Inet4Address.getLoopbackAddress(), + command, "", "", + null, null, null); + } + + @Test + public void testDebugCommandNotLoggedAtInfo() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.INFO, Optional.of(Arrays.asList(DUMMY_COMMAND_1))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + verify(logger, never()).logAuditMessage(anyString()); + } + + @Test + public void testDebugCommandLoggedAtDebug() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.DEBUG, Optional.of(Arrays.asList(DUMMY_COMMAND_1))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + verify(logger, times(1)).logAuditMessage(anyString()); + } + + @Test + public void testInfoCommandLoggedAtInfo() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.INFO, Optional.of(Arrays.asList(DUMMY_COMMAND_1))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, times(1)).logAuditMessage(anyString()); + } + + @Test + public void testMultipleDebugCommandsNotLoggedAtInfo() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.INFO, + Optional.of(Arrays.asList(DUMMY_COMMAND_1, DUMMY_COMMAND_2))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, never()).logAuditMessage(anyString()); + } + + @Test + public void testMultipleDebugCommandsLoggedAtDebug() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.DEBUG, + Optional.of(Arrays.asList(DUMMY_COMMAND_1, DUMMY_COMMAND_2))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, times(2)).logAuditMessage(anyString()); + } + + @Test + public void testEmptyDebugCommands() { + DefaultAuditLogger logger = makeSpyLogger( + Level.INFO, Optional.>absent()); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, times(2)).logAuditMessage(anyString()); + } +}