Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 49766 invoked from network); 4 Mar 2011 04:44:10 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Mar 2011 04:44:10 -0000 Received: (qmail 99632 invoked by uid 500); 4 Mar 2011 04:44:10 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 99587 invoked by uid 500); 4 Mar 2011 04:44:10 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 99572 invoked by uid 99); 4 Mar 2011 04:44:09 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 04:44:09 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 04:44:05 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id EED5A23888CD; Fri, 4 Mar 2011 04:43:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1077680 - in /hadoop/common/branches/branch-0.20-security-patches: ./ src/core/org/apache/hadoop/io/ src/core/org/apache/hadoop/io/nativeio/ src/mapred/org/apache/hadoop/mapred/ src/native/ src/native/src/ src/native/src/org/apache/hadoop/... Date: Fri, 04 Mar 2011 04:43:44 -0000 To: common-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110304044344.EED5A23888CD@eris.apache.org> Author: omalley Date: Fri Mar 4 04:43:43 2011 New Revision: 1077680 URL: http://svn.apache.org/viewvc?rev=1077680&view=rev Log: commit 9eb13a357be8202418bbf2bb8b80f669c738f99c Author: Devaraj Das Date: Fri Sep 17 00:24:41 2010 -0700 : Fixes task log servlet vulnerabilities via symlinks +++ b/YAHOO-CHANGES.txt + : Fixes task log servlet vulnerabilities via symlinks. + (Todd Lipcon and Devaraj Das) + + , : Write task initialization to avoid race conditions + leading to privilege escalation and resource leakage by performing more acti + as the user. Owen O'Malley, Devaraj Das, Chris Douglas + Added: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/SecureIOUtils.java hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/Errno.java hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIO.java hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIOException.java hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.c hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.h hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/TestSecureIOUtils.java hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/nativeio/ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java Modified: hadoop/common/branches/branch-0.20-security-patches/build.xml hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/IndexCache.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/MapTask.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/SpillRecord.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLog.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogServlet.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java hadoop/common/branches/branch-0.20-security-patches/src/native/Makefile.am hadoop/common/branches/branch-0.20-security-patches/src/native/config.h.in hadoop/common/branches/branch-0.20-security-patches/src/native/configure.ac hadoop/common/branches/branch-0.20-security-patches/src/native/src/org_apache_hadoop.h hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestIndexCache.java Modified: hadoop/common/branches/branch-0.20-security-patches/build.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/build.xml?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/build.xml (original) +++ hadoop/common/branches/branch-0.20-security-patches/build.xml Fri Mar 4 04:43:43 2011 @@ -548,6 +548,7 @@ + + + + + * An example of such an attack is: + *
    + *
  1. Malicious user removes his task's syslog file, and puts a link to the + * jobToken file of a target user.
    2. + *
  2. Malicious user tries to open the syslog file via the servlet on the + * tasktracker.
    3. + *
  3. The tasktracker is unaware of the symlink, and simply streams the contents + * of the jobToken file. The malicious user can now access potentially sensitive + * map outputs, etc. of the target user's job.
    4. + *
+ * A similar attack is possible involving the + * {@link org.apache.hadoop.mapred.TaskLogsTruncater}, but here it is to do + * with unsecure write to a file. + *
+ */ +public class SecureIOUtils { + + /** + * Ensure that we are set up to run with the appropriate native support code. + * If security is disabled, and the support code is unavailable, this class + * still tries its best to be secure, but is vulnerable to some race condition + * attacks. + * + * If security is enabled but the support code is unavailable, throws a + * RuntimeException since we don't want to run insecurely. + */ + static { + boolean shouldBeSecure = UserGroupInformation.isSecurityEnabled(); + boolean canBeSecure = NativeIO.isAvailable(); + + if (!canBeSecure && shouldBeSecure) { + throw new RuntimeException( + "Secure IO is not possible without native code extensions."); + } + + // Pre-cache an instance of the raw FileSystem since we sometimes + // do secure IO in a shutdown hook, where this call could fail. + try { + rawFilesystem = FileSystem.getLocal(new Configuration()).getRaw(); + } catch (IOException ie) { + throw new RuntimeException( + "Couldn't obtain an instance of RawLocalFileSystem."); + } + + // SecureIO just skips security checks in the case that security is + // disabled + skipSecurity = !canBeSecure; + } + + private final static boolean skipSecurity; + private final static FileSystem rawFilesystem; + + /** + * Open the given File for read access, verifying the expected user/group + * constraints. + * @param f the file that we are trying to open + * @param expectedOwner the expected user owner for the file + * @param expectedGroup the expected group owner for the file + * @throws IOException if an IO Error occurred, or the user/group does not + * match + */ + public static FileInputStream openForRead(File f, String expectedOwner, + String expectedGroup) throws IOException { + if (skipSecurity) { + // Subject to race conditions but this is the best we can do + FileStatus status = + rawFilesystem.getFileStatus(new Path(f.getAbsolutePath())); + checkStat(f, status.getOwner(), status.getGroup(), + expectedOwner, expectedGroup); + return new FileInputStream(f); + } + + FileInputStream fis = new FileInputStream(f); + boolean success = false; + try { + Stat stat = NativeIO.fstat(fis.getFD()); + checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner, + expectedGroup); + success = true; + return fis; + } finally { + if (!success) fis.close(); + } + } + + private static FileOutputStream insecureCreateForWrite(File f, + int permissions) throws IOException { + // If we can't do real security, do a racy exists check followed by an + // open and chmod + if (f.exists()) { + throw new AlreadyExistsException("File " + f + " already exists"); + } + FileOutputStream fos = new FileOutputStream(f); + boolean success = false; + try { + rawFilesystem.setPermission(new Path(f.getAbsolutePath()), + new FsPermission((short)permissions)); + success = true; + return fos; + } finally { + if (!success) { + fos.close(); + } + } + } + + /** + * Open the specified File for write access, ensuring that it does not exist. + * @param f the file that we want to create + * @param permissions we want to have on the file (if security is enabled) + * + * @throws AlreadyExistsException if the file already exists + * @throws IOException if any other error occurred + */ + public static FileOutputStream createForWrite(File f, int permissions) + throws IOException { + if (skipSecurity) { + return insecureCreateForWrite(f, permissions); + } else { + // Use the native wrapper around open(2) + try { + FileDescriptor fd = NativeIO.open(f.getAbsolutePath(), + NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL, + permissions); + return new FileOutputStream(fd); + } catch (NativeIOException nioe) { + if (nioe.getErrno() == Errno.EEXIST) { + throw new AlreadyExistsException(nioe); + } + throw nioe; + } + } + } + + private static void checkStat(File f, String owner, String group, + String expectedOwner, + String expectedGroup) throws IOException { + if (expectedOwner != null && + !expectedOwner.equals(owner)) { + throw new IOException( + "Owner '" + owner + "' for path " + f + " did not match " + + "expected owner '" + expectedOwner + "'"); + } + if (expectedGroup != null && + !expectedGroup.equals(group)) { + throw new IOException( + "Group '" + group + "' for path " + f + " did not match " + + "expected group '" + expectedGroup + "'"); + } + } + + /** + * Signals that an attempt to create a file at a given pathname has failed + * because another file already existed at that path. + */ + public static class AlreadyExistsException extends IOException { + private static final long serialVersionUID = -6615764817774423232L; + public AlreadyExistsException(String msg) { + super(msg); + } + + public AlreadyExistsException(Throwable cause) { + super(cause); + } + } +} Added: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/Errno.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/Errno.java?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/Errno.java (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/Errno.java Fri Mar 4 04:43:43 2011 @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.nativeio; + +/** + * Enum representing POSIX errno values. + */ +public enum Errno { + EPERM, + ENOENT, + ESRCH, + EINTR, + EIO, + ENXIO, + E2BIG, + ENOEXEC, + EBADF, + ECHILD, + EAGAIN, + ENOMEM, + EACCES, + EFAULT, + ENOTBLK, + EBUSY, + EEXIST, + EXDEV, + ENODEV, + ENOTDIR, + EISDIR, + EINVAL, + ENFILE, + EMFILE, + ENOTTY, + ETXTBSY, + EFBIG, + ENOSPC, + ESPIPE, + EROFS, + EMLINK, + EPIPE, + EDOM, + ERANGE, + + UNKNOWN; +} Added: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIO.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIO.java (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIO.java Fri Mar 4 04:43:43 2011 @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.nativeio; + +import java.io.FileDescriptor; +import java.io.IOException; + +import org.apache.hadoop.util.NativeCodeLoader; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +/** + * JNI wrappers for various native IO-related calls not available in Java. + * These functions should generally be used alongside a fallback to another + * more portable mechanism. + */ +public class NativeIO { + // Flags for open() call from bits/fcntl.h + public static final int O_RDONLY = 00; + public static final int O_WRONLY = 01; + public static final int O_RDWR = 02; + public static final int O_CREAT = 0100; + public static final int O_EXCL = 0200; + public static final int O_NOCTTY = 0400; + public static final int O_TRUNC = 01000; + public static final int O_APPEND = 02000; + public static final int O_NONBLOCK = 04000; + public static final int O_SYNC = 010000; + public static final int O_ASYNC = 020000; + public static final int O_FSYNC = O_SYNC; + public static final int O_NDELAY = O_NONBLOCK; + + private static final Log LOG = LogFactory.getLog(NativeIO.class); + + private static boolean nativeLoaded = false; + + static { + if (NativeCodeLoader.isNativeCodeLoaded()) { + try { + initNative(); + nativeLoaded = true; + } catch (Throwable t) { + // This can happen if the user has an older version of libhadoop.so + // installed - in this case we can continue without native IO + // after warning + LOG.error("Unable to initialize NativeIO libraries", t); + } + } + } + + /** + * Return true if the JNI-based native IO extensions are available. + */ + public static boolean isAvailable() { + return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; + } + + /** Wrapper around open(2) */ + public static native FileDescriptor open(String path, int flags, int mode) throws IOException; + /** Wrapper around fstat(2) */ + public static native Stat fstat(FileDescriptor fd) throws IOException; + /** Initialize the JNI method ID and class ID cache */ + private static native void initNative(); + + + /** + * Result type of the fstat call + */ + public static class Stat { + private String owner, group; + private int mode; + + // Mode constants + public static final int S_IFMT = 0170000; /* type of file */ + public static final int S_IFIFO = 0010000; /* named pipe (fifo) */ + public static final int S_IFCHR = 0020000; /* character special */ + public static final int S_IFDIR = 0040000; /* directory */ + public static final int S_IFBLK = 0060000; /* block special */ + public static final int S_IFREG = 0100000; /* regular */ + public static final int S_IFLNK = 0120000; /* symbolic link */ + public static final int S_IFSOCK = 0140000; /* socket */ + public static final int S_IFWHT = 0160000; /* whiteout */ + public static final int S_ISUID = 0004000; /* set user id on execution */ + public static final int S_ISGID = 0002000; /* set group id on execution */ + public static final int S_ISVTX = 0001000; /* save swapped text even after use */ + public static final int S_IRUSR = 0000400; /* read permission, owner */ + public static final int S_IWUSR = 0000200; /* write permission, owner */ + public static final int S_IXUSR = 0000100; /* execute/search permission, owner */ + + Stat(String owner, String group, int mode) { + this.owner = owner; + this.group = group; + this.mode = mode; + } + + public String toString() { + return "Stat(owner='" + owner + "', group='" + group + "'" + + ", mode=" + mode + ")"; + } + + public String getOwner() { + return owner; + } + public String getGroup() { + return group; + } + public int getMode() { + return mode; + } + } +} Added: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIOException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIOException.java?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIOException.java (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/io/nativeio/NativeIOException.java Fri Mar 4 04:43:43 2011 @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.nativeio; + +import java.io.IOException; + +/** + * An exception generated by a call to the native IO code. + * + * These exceptions simply wrap errno result codes. + */ +public class NativeIOException extends IOException { + private static final long serialVersionUID = -6615764817732323232L; + private Errno errno; + + public NativeIOException(String msg, Errno errno) { + super(msg); + this.errno = errno; + } + + public Errno getErrno() { + return errno; + } + + public String toString() { + return errno.toString() + ": " + super.getMessage(); + } +} + + Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/IndexCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/IndexCache.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/IndexCache.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/IndexCache.java Fri Mar 4 04:43:43 2011 @@ -53,16 +53,17 @@ class IndexCache { * @param reduce * @param fileName The file to read the index information from if it is not * already present in the cache + * @param expectedIndexOwner The expected owner of the index file * @return The Index Information * @throws IOException */ public IndexRecord getIndexInformation(String mapId, int reduce, - Path fileName) throws IOException { + Path fileName, String expectedIndexOwner) throws IOException { IndexInformation info = cache.get(mapId); if (info == null) { - info = readIndexFileToCache(fileName, mapId); + info = readIndexFileToCache(fileName, mapId, expectedIndexOwner); } else { synchronized (info) { while (null == info.mapSpillRecord) { @@ -86,7 +87,7 @@ class IndexCache { } private IndexInformation readIndexFileToCache(Path indexFileName, - String mapId) throws IOException { + String mapId, String expectedIndexOwner) throws IOException { IndexInformation info; IndexInformation newInd = new IndexInformation(); if ((info = cache.putIfAbsent(mapId, newInd)) != null) { @@ -105,7 +106,7 @@ class IndexCache { LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ; SpillRecord tmp = null; try { - tmp = new SpillRecord(indexFileName, conf); + tmp = new SpillRecord(indexFileName, conf, expectedIndexOwner); } catch (Throwable e) { tmp = new SpillRecord(0); cache.remove(mapId); Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java Fri Mar 4 04:43:43 2011 @@ -290,5 +290,10 @@ class LinuxTaskController extends TaskCo } } } + + @Override + public String getRunAsUser(JobConf conf) { + return conf.getUser(); + } } Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/MapTask.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/MapTask.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/MapTask.java Fri Mar 4 04:43:43 2011 @@ -65,6 +65,7 @@ import org.apache.hadoop.mapreduce.split import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.IndexedSortable; import org.apache.hadoop.util.IndexedSorter; import org.apache.hadoop.util.Progress; @@ -1506,7 +1507,8 @@ class MapTask extends Task { // read in paged indices for (int i = indexCacheList.size(); i < numSpills; ++i) { Path indexFileName = mapOutputFile.getSpillIndexFile(i); - indexCacheList.add(new SpillRecord(indexFileName, job)); + indexCacheList.add(new SpillRecord(indexFileName, job, + UserGroupInformation.getCurrentUser().getShortUserName())); } //make correction in the length to include the sequence file header Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/SpillRecord.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/SpillRecord.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/SpillRecord.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/SpillRecord.java Fri Mar 4 04:43:43 2011 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.mapred; +import java.io.DataInputStream; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.LongBuffer; @@ -26,11 +28,11 @@ import java.util.zip.CheckedOutputStream import java.util.zip.Checksum; import org.apache.hadoop.fs.ChecksumException; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.SecureIOUtils; import static org.apache.hadoop.mapred.MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH; @@ -47,15 +49,18 @@ class SpillRecord { entries = buf.asLongBuffer(); } - public SpillRecord(Path indexFileName, JobConf job) throws IOException { - this(indexFileName, job, new CRC32()); + public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner) + throws IOException { + this(indexFileName, job, new CRC32(), expectedIndexOwner); } - public SpillRecord(Path indexFileName, JobConf job, Checksum crc) - throws IOException { + public SpillRecord(Path indexFileName, JobConf job, Checksum crc, + String expectedIndexOwner) throws IOException { final FileSystem rfs = FileSystem.getLocal(job).getRaw(); - final FSDataInputStream in = rfs.open(indexFileName); + final DataInputStream in = + new DataInputStream(SecureIOUtils.openForRead( + new File(indexFileName.toUri().getPath()), expectedIndexOwner, null)); try { final long length = rfs.getFileStatus(indexFileName).getLen(); final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH; Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java Fri Mar 4 04:43:43 2011 @@ -169,6 +169,13 @@ public abstract class TaskController imp } } } + + /** + * Returns the local unix user that a given job will run as. + */ + public String getRunAsUser(JobConf conf) { + return System.getProperty("user.name"); + } //Write the JVM command line to a file under the specified directory // Note that the JVM will be launched using a setuid executable, and Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLog.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLog.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLog.java Fri Mar 4 04:43:43 2011 @@ -26,6 +26,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; @@ -39,8 +40,10 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.util.ProcessTree; import org.apache.hadoop.util.Shell; @@ -106,7 +109,8 @@ public class TaskLog { new HashMap(); File indexFile = getIndexFile(taskid, isCleanup); - BufferedReader fis = new BufferedReader(new java.io.FileReader(indexFile)); + BufferedReader fis = new BufferedReader(new InputStreamReader( + SecureIOUtils.openForRead(indexFile, obtainLogDirOwner(taskid), null))); //the format of the index file is //LOG_DIR: //stdout: @@ -153,6 +157,18 @@ public class TaskLog { return new File(getAttemptDir(taskid, isCleanup), "log.index"); } + /** + * Obtain the owner of the log dir. This is + * determined by checking the job's log directory. + */ + static String obtainLogDirOwner(TaskAttemptID taskid) throws IOException { + Configuration conf = new Configuration(); + FileSystem raw = FileSystem.getLocal(conf).getRaw(); + Path jobLogDir = new Path(getJobDir(taskid.getJobID()).getAbsolutePath()); + FileStatus jobStat = raw.getFileStatus(jobLogDir); + return jobStat.getOwner(); + } + static String getBaseLogDir() { return System.getProperty("hadoop.log.dir"); } @@ -194,7 +210,8 @@ public class TaskLog { File tmpIndexFile = getTmpIndexFile(currentTaskid, isCleanup); BufferedOutputStream bos = - new BufferedOutputStream(new FileOutputStream(tmpIndexFile,false)); + new BufferedOutputStream( + SecureIOUtils.createForWrite(tmpIndexFile, 0644)); DataOutputStream dos = new DataOutputStream(bos); //the format of the index file is //LOG_DIR: @@ -330,7 +347,9 @@ public class TaskLog { start += fileDetail.start; end += fileDetail.start; bytesRemaining = end - start; - file = new FileInputStream(new File(fileDetail.location, kind.toString())); + String owner = obtainLogDirOwner(taskid); + file = SecureIOUtils.openForRead(new File(fileDetail.location, kind.toString()), + owner, null); // skip upto start long pos = 0; while (pos < start) { Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogServlet.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogServlet.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogServlet.java Fri Mar 4 04:43:43 2011 @@ -28,6 +28,8 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.QueueManager.QueueACL; import org.apache.hadoop.mapreduce.JobACL; @@ -43,6 +45,9 @@ import org.apache.hadoop.util.StringUtil public class TaskLogServlet extends HttpServlet { private static final long serialVersionUID = -6615764817774487321L; + private static final Log LOG = + LogFactory.getLog(TaskLog.class); + private boolean haveTaskLog(TaskAttemptID taskId, boolean isCleanup, TaskLog.LogName type) { File f = TaskLog.getTaskLogFile(taskId, isCleanup, type); @@ -101,11 +106,10 @@ public class TaskLogServlet extends Http // do nothing } else { - response.sendError(HttpServletResponse.SC_GONE, - "Failed to retrieve " + filter + " log for task: " + - taskId); - out.write(("TaskLogServlet exception:\n" + - StringUtils.stringifyException(ioe) + "\n").getBytes()); + String msg = "Failed to retrieve " + filter + " log for task: " + + taskId; + LOG.warn(msg, ioe); + response.sendError(HttpServletResponse.SC_GONE, msg); } } } Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskLogsTruncater.java Fri Mar 4 04:43:43 2011 @@ -20,7 +20,6 @@ package org.apache.hadoop.mapred; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.util.HashMap; @@ -32,6 +31,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapred.TaskLog.LogName; import org.apache.hadoop.mapred.TaskLog.LogFileDetail; @@ -74,6 +74,15 @@ public class TaskLogsTruncater { * retainSize. */ public void truncateLogs(JVMInfo lInfo) { + Task firstAttempt = lInfo.getAllAttempts().get(0); + String owner; + try { + owner = TaskLog.obtainLogDirOwner(firstAttempt.getTaskID()); + } catch (IOException ioe) { + LOG.error("Unable to create a secure IO context to truncate logs for " + + firstAttempt, ioe); + return; + } // Read the log-file details for all the attempts that ran in this JVM Map> taskLogFileDetails; @@ -120,7 +129,7 @@ public class TaskLogsTruncater { // ////// Open truncate.tmp file for writing ////// File tmpFile = new File(attemptLogDir, "truncate.tmp"); try { - tmpFileOutputStream = new FileOutputStream(tmpFile); + tmpFileOutputStream = SecureIOUtils.createForWrite(tmpFile, 0644); } catch (IOException ioe) { LOG.warn("Cannot open " + tmpFile.getAbsolutePath() + " for writing truncated log-file " @@ -132,11 +141,11 @@ public class TaskLogsTruncater { // ////// Open logFile for reading ////// try { - logFileInputStream = new FileInputStream(logFile); - } catch (FileNotFoundException fe) { + logFileInputStream = SecureIOUtils.openForRead(logFile, owner, null); + } catch (IOException ioe) { if (LOG.isDebugEnabled()) { LOG.debug("Cannot open " + logFile.getAbsolutePath() - + " for reading. Continuing with other log files"); + + " for reading. Continuing with other log files", ioe); } try { tmpFileOutputStream.close(); @@ -275,8 +284,8 @@ public class TaskLogsTruncater { /** * Get the logFileDetails of all the list of attempts passed. + * @param allAttempts the attempts we are interested in * - * @param lInfo * @return a map of task to the log-file detail * @throws IOException */ @@ -287,8 +296,7 @@ public class TaskLogsTruncater { for (Task task : allAttempts) { Map allLogsFileDetails; allLogsFileDetails = - TaskLog.getAllLogsFileDetails(task.getTaskID(), - task.isTaskCleanupTask()); + TaskLog.getAllLogsFileDetails(task.getTaskID(), task.isTaskCleanupTask()); taskLogFileDetails.put(task, allLogsFileDetails); } return taskLogFileDetails; Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Fri Mar 4 04:43:43 2011 @@ -18,6 +18,7 @@ package org.apache.hadoop.mapred; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -70,6 +71,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.Server; @@ -2897,6 +2899,15 @@ public class TaskTracker implements MRCo return task.getTaskID().hashCode(); } } + + private void authorizeJVM(JobID jobId) throws IOException { + String currentJobId = + UserGroupInformation.getCurrentUser().getUserName(); + if (!currentJobId.equals(jobId.toString())) { + throw new IOException ("JVM with " + currentJobId + + " is not authorized for " + jobId); + } + } // /////////////////////////////////////////////////////////////// @@ -2908,6 +2919,7 @@ public class TaskTracker implements MRCo */ public synchronized JvmTask getTask(JvmContext context) throws IOException { + authorizeJVM(context.jvmId.getJobId()); JVMId jvmId = context.jvmId; LOG.debug("JVM with ID : " + jvmId + " asked for a task"); // save pid of task JVM sent by child @@ -2948,6 +2960,7 @@ public class TaskTracker implements MRCo public synchronized boolean statusUpdate(TaskAttemptID taskid, TaskStatus taskStatus) throws IOException { + authorizeJVM(taskid.getJobID()); TaskInProgress tip = tasks.get(taskid); if (tip != null) { tip.reportProgress(taskStatus); @@ -2963,6 +2976,7 @@ public class TaskTracker implements MRCo * diagnostic info */ public synchronized void reportDiagnosticInfo(TaskAttemptID taskid, String info) throws IOException { + authorizeJVM(taskid.getJobID()); TaskInProgress tip = tasks.get(taskid); if (tip != null) { tip.reportDiagnosticInfo(info); @@ -2973,6 +2987,7 @@ public class TaskTracker implements MRCo public synchronized void reportNextRecordRange(TaskAttemptID taskid, SortedRanges.Range range) throws IOException { + authorizeJVM(taskid.getJobID()); TaskInProgress tip = tasks.get(taskid); if (tip != null) { tip.reportNextRecordRange(range); @@ -2984,6 +2999,7 @@ public class TaskTracker implements MRCo /** Child checking to see if we're alive. Normally does nothing.*/ public synchronized boolean ping(TaskAttemptID taskid) throws IOException { + authorizeJVM(taskid.getJobID()); return tasks.get(taskid) != null; } @@ -2994,6 +3010,7 @@ public class TaskTracker implements MRCo public synchronized void commitPending(TaskAttemptID taskid, TaskStatus taskStatus) throws IOException { + authorizeJVM(taskid.getJobID()); LOG.info("Task " + taskid + " is in commit-pending," +"" + " task state:" +taskStatus.getRunState()); statusUpdate(taskid, taskStatus); @@ -3003,7 +3020,9 @@ public class TaskTracker implements MRCo /** * Child checking whether it can commit */ - public synchronized boolean canCommit(TaskAttemptID taskid) { + public synchronized boolean canCommit(TaskAttemptID taskid) + throws IOException { + authorizeJVM(taskid.getJobID()); return commitResponses.contains(taskid); //don't remove it now } @@ -3012,6 +3031,7 @@ public class TaskTracker implements MRCo */ public synchronized void done(TaskAttemptID taskid) throws IOException { + authorizeJVM(taskid.getJobID()); TaskInProgress tip = tasks.get(taskid); commitResponses.remove(taskid); if (tip != null) { @@ -3027,6 +3047,7 @@ public class TaskTracker implements MRCo */ public synchronized void shuffleError(TaskAttemptID taskId, String message) throws IOException { + authorizeJVM(taskId.getJobID()); LOG.fatal("Task: " + taskId + " - Killed due to Shuffle Failure: " + message); TaskInProgress tip = runningTasks.get(taskId); tip.reportDiagnosticInfo("Shuffle Error: " + message); @@ -3038,6 +3059,7 @@ public class TaskTracker implements MRCo */ public synchronized void fsError(TaskAttemptID taskId, String message) throws IOException { + authorizeJVM(taskId.getJobID()); LOG.fatal("Task: " + taskId + " - Killed due to FSError: " + message); TaskInProgress tip = runningTasks.get(taskId); tip.reportDiagnosticInfo("FSError: " + message); @@ -3049,6 +3071,7 @@ public class TaskTracker implements MRCo */ public synchronized void fatalError(TaskAttemptID taskId, String msg) throws IOException { + authorizeJVM(taskId.getJobID()); LOG.fatal("Task: " + taskId + " - Killed : " + msg); TaskInProgress tip = runningTasks.get(taskId); tip.reportDiagnosticInfo("Error: " + msg); @@ -3058,6 +3081,7 @@ public class TaskTracker implements MRCo public synchronized MapTaskCompletionEventsUpdate getMapCompletionEvents( JobID jobId, int fromEventId, int maxLocs, TaskAttemptID id) throws IOException { + authorizeJVM(jobId); TaskCompletionEvent[]mapEvents = TaskCompletionEvent.EMPTY_ARRAY; synchronized (shouldReset) { if (shouldReset.remove(id)) { @@ -3316,7 +3340,7 @@ public class TaskTracker implements MRCo // true iff IOException was caused by attempt to access input boolean isInputException = true; OutputStream outStream = null; - FSDataInputStream mapOutputIn = null; + FileInputStream mapOutputIn = null; long totalRead = 0; ShuffleServerInstrumentation shuffleMetrics = @@ -3339,12 +3363,14 @@ public class TaskTracker implements MRCo context.getAttribute("local.file.system")).getRaw(); String userName = null; + String runAsUserName = null; synchronized (tracker.runningJobs) { RunningJob rjob = tracker.runningJobs.get(JobID.forName(jobId)); if (rjob == null) { throw new IOException("Unknown job " + jobId + "!!"); } userName = rjob.jobConf.getUser(); + runAsUserName = tracker.getTaskController().getRunAsUser(rjob.jobConf); } // Index file Path indexFileName = @@ -3363,7 +3389,8 @@ public class TaskTracker implements MRCo * the map-output for the given reducer is available. */ IndexRecord info = - tracker.indexCache.getIndexInformation(mapId, reduce,indexFileName); + tracker.indexCache.getIndexInformation(mapId, reduce,indexFileName, + runAsUserName); //set the custom "from-map-task" http header to the map task from which //the map output data is being transferred @@ -3391,10 +3418,11 @@ public class TaskTracker implements MRCo * send it to the reducer. */ //open the map-output file - mapOutputIn = rfs.open(mapOutputFileName); + mapOutputIn = SecureIOUtils.openForRead( + new File(mapOutputFileName.toUri().getPath()), runAsUserName, null); //seek to the correct offset for the reduce - mapOutputIn.seek(info.startOffset); + mapOutputIn.skip(info.startOffset); long rem = info.partLength; int len = mapOutputIn.read(buffer, 0, (int)Math.min(rem, MAX_BYTES_TO_READ)); Modified: hadoop/common/branches/branch-0.20-security-patches/src/native/Makefile.am URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/Makefile.am?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/Makefile.am (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/Makefile.am Fri Mar 4 04:43:43 2011 @@ -33,6 +33,7 @@ export PLATFORM = $(shell echo $$OS_NAME AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \ -Isrc/org/apache/hadoop/io/compress/zlib \ + -Isrc/org/apache/hadoop/io/nativeio \ -Isrc/org/apache/hadoop/security AM_LDFLAGS = @JNI_LDFLAGS@ -m$(JVM_DATA_MODEL) AM_CFLAGS = -g -Wall -fPIC -O2 -m$(JVM_DATA_MODEL) @@ -42,7 +43,10 @@ libhadoop_la_SOURCES = src/org/apache/ha src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \ src/org/apache/hadoop/security/getGroup.c \ src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \ - src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c + src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c \ + src/org/apache/hadoop/io/nativeio/file_descriptor.c \ + src/org/apache/hadoop/io/nativeio/errno_enum.c \ + src/org/apache/hadoop/io/nativeio/NativeIO.c libhadoop_la_LDFLAGS = -version-info 1:0:0 libhadoop_la_LIBADD = -ldl -ljvm Modified: hadoop/common/branches/branch-0.20-security-patches/src/native/config.h.in URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/config.h.in?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/config.h.in (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/config.h.in Fri Mar 4 04:43:43 2011 @@ -3,6 +3,10 @@ /* The 'actual' dynamic-library for '-lz' */ #undef HADOOP_ZLIB_LIBRARY +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#undef HAVE_DECL_STRERROR_R + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H @@ -39,6 +43,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H +/* Define to 1 if you have the `strerror_r' function. */ +#undef HAVE_STRERROR_R + /* Define to 1 if you have the header file. */ #undef HAVE_STRINGS_H @@ -81,6 +88,9 @@ /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS +/* Define to 1 if strerror_r returns char *. */ +#undef STRERROR_R_CHAR_P + /* Version number of package */ #undef VERSION Modified: hadoop/common/branches/branch-0.20-security-patches/src/native/configure.ac URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/configure.ac?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/configure.ac (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/configure.ac Fri Mar 4 04:43:43 2011 @@ -95,6 +95,9 @@ AC_C_CONST # Checks for library functions. AC_CHECK_FUNCS([memset]) +# Check for nonstandard STRERROR_R +AC_FUNC_STRERROR_R + AC_CONFIG_FILES([Makefile]) AC_OUTPUT Added: hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/NativeIO.c Fri Mar 4 04:43:43 2011 @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_io_nativeio_NativeIO.h" +#include "file_descriptor.h" +#include "errno_enum.h" + +// the NativeIO$Stat inner class and its constructor +static jclass stat_clazz; +static jmethodID stat_ctor; + +// the NativeIOException class and its constructor +static jclass nioe_clazz; +static jmethodID nioe_ctor; + +// Internal functions +static void throw_ioe(JNIEnv* env, int errnum); +static ssize_t get_pw_buflen(); + + +static void stat_init(JNIEnv *env) { + // Init Stat + jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$Stat"); + PASS_EXCEPTIONS(env); + stat_clazz = (*env)->NewGlobalRef(env, clazz); + stat_ctor = (*env)->GetMethodID(env, stat_clazz, "", + "(Ljava/lang/String;Ljava/lang/String;I)V"); +} + +static void stat_deinit(JNIEnv *env) { + if (stat_clazz != NULL) { + (*env)->DeleteGlobalRef(env, stat_clazz); + stat_clazz = NULL; + } +} + +static void nioe_init(JNIEnv *env) { + // Init NativeIOException + nioe_clazz = (*env)->FindClass( + env, "org/apache/hadoop/io/nativeio/NativeIOException"); + PASS_EXCEPTIONS(env); + + nioe_clazz = (*env)->NewGlobalRef(env, nioe_clazz); + nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "", + "(Ljava/lang/String;Lorg/apache/hadoop/io/nativeio/Errno;)V"); +} + +static void nioe_deinit(JNIEnv *env) { + if (nioe_clazz != NULL) { + (*env)->DeleteGlobalRef(env, nioe_clazz); + nioe_clazz = NULL; + } + nioe_ctor = NULL; +} + +/* + * private static native void initNative(); + * + * We rely on this function rather than lazy initialization because + * the lazy approach may have a race if multiple callers try to + * init at the same time. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_initNative( + JNIEnv *env, jclass clazz) { + + stat_init(env); + PASS_EXCEPTIONS_GOTO(env, error); + nioe_init(env); + PASS_EXCEPTIONS_GOTO(env, error); + fd_init(env); + PASS_EXCEPTIONS_GOTO(env, error); + errno_enum_init(env); + PASS_EXCEPTIONS_GOTO(env, error); + return; +error: + // these are all idempodent and safe to call even if the + // class wasn't initted yet + stat_deinit(env); + nioe_deinit(env); + fd_deinit(env); + errno_enum_deinit(env); +} + +/* + * public static native Stat fstat(FileDescriptor fd); + */ +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_fstat( + JNIEnv *env, jclass clazz, jobject fd_object) +{ + jobject ret = NULL; + char *pw_buf = NULL; + + int fd = fd_get(env, fd_object); + PASS_EXCEPTIONS_GOTO(env, cleanup); + + struct stat s; + int rc = fstat(fd, &s); + if (rc != 0) { + throw_ioe(env, errno); + goto cleanup; + } + + size_t pw_buflen = get_pw_buflen(); + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + + // Grab username + struct passwd pwd, *pwdp; + while ((rc = getpwuid_r(s.st_uid, &pwd, pw_buf, pw_buflen, &pwdp)) != 0) { + if (rc != ERANGE) { + throw_ioe(env, rc); + goto cleanup; + } + free(pw_buf); + pw_buflen *= 2; + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + } + assert(pwdp == &pwd); + + jstring jstr_username = (*env)->NewStringUTF(env, pwd.pw_name); + if (jstr_username == NULL) goto cleanup; + + // Grab group + struct group grp, *grpp; + while ((rc = getgrgid_r(s.st_gid, &grp, pw_buf, pw_buflen, &grpp)) != 0) { + if (rc != ERANGE) { + throw_ioe(env, rc); + goto cleanup; + } + free(pw_buf); + pw_buflen *= 2; + if ((pw_buf = malloc(pw_buflen)) == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer"); + goto cleanup; + } + } + assert(grpp == &grp); + + jstring jstr_groupname = (*env)->NewStringUTF(env, grp.gr_name); + PASS_EXCEPTIONS_GOTO(env, cleanup); + + // Construct result + ret = (*env)->NewObject(env, stat_clazz, stat_ctor, + jstr_username, jstr_groupname, s.st_mode); + +cleanup: + if (pw_buf != NULL) free(pw_buf); + return ret; +} + + +/* + * public static native FileDescriptor open(String path, int flags, int mode); + */ +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_open( + JNIEnv *env, jclass clazz, jstring j_path, + jint flags, jint mode) +{ + jobject ret = NULL; + + const char *path = (*env)->GetStringUTFChars(env, j_path, NULL); + if (path == NULL) goto cleanup; // JVM throws Exception for us + + int fd; + if (flags & O_CREAT) { + fd = open(path, flags, mode); + } else { + fd = open(path, flags); + } + + if (fd == -1) { + throw_ioe(env, errno); + goto cleanup; + } + + ret = fd_create(env, fd); + +cleanup: + if (path != NULL) { + (*env)->ReleaseStringUTFChars(env, j_path, path); + } + return ret; +} + +/* + * Throw a java.IO.IOException, generating the message from errno. + */ +static void throw_ioe(JNIEnv* env, int errnum) +{ + const char* message; + char buffer[80]; + jstring jstr_message; + + buffer[0] = 0; +#ifdef STRERROR_R_CHAR_P + // GNU strerror_r + message = strerror_r(errnum, buffer, sizeof(buffer)); + assert (message != NULL); +#else + int ret = strerror_r(errnum, buffer, sizeof(buffer)); + if (ret == 0) { + message = buffer; + } else { + message = "Unknown error"; + } +#endif + jobject errno_obj = errno_to_enum(env, errnum); + + if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL) + goto err; + + jthrowable obj = (jthrowable)(*env)->NewObject(env, nioe_clazz, nioe_ctor, + jstr_message, errno_obj); + if (obj == NULL) goto err; + + (*env)->Throw(env, obj); + return; + +err: + if (jstr_message != NULL) + (*env)->ReleaseStringUTFChars(env, jstr_message, message); +} + + +/* + * Determine how big a buffer we need for reentrant getpwuid_r and getgrnam_r + */ +ssize_t get_pw_buflen() { + size_t ret = 0; + #ifdef _SC_GETPW_R_SIZE_MAX + ret = sysconf(_SC_GETPW_R_SIZE_MAX); + #endif + return (ret > 512) ? ret : 512; +} +/** + * vim: sw=2: ts=2: et: + */ + Added: hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.c?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.c (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.c Fri Mar 4 04:43:43 2011 @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include + #include + #include + +#include "org_apache_hadoop.h" + +typedef struct errno_mapping { + int errno_val; + char *errno_str; +} errno_mapping_t; + +#define MAPPING(x) {x, #x} +static errno_mapping_t ERRNO_MAPPINGS[] = { + MAPPING(EPERM), + MAPPING(ENOENT), + MAPPING(ESRCH), + MAPPING(EINTR), + MAPPING(EIO), + MAPPING(ENXIO), + MAPPING(E2BIG), + MAPPING(ENOEXEC), + MAPPING(EBADF), + MAPPING(ECHILD), + MAPPING(EAGAIN), + MAPPING(ENOMEM), + MAPPING(EACCES), + MAPPING(EFAULT), + MAPPING(ENOTBLK), + MAPPING(EBUSY), + MAPPING(EEXIST), + MAPPING(EXDEV), + MAPPING(ENODEV), + MAPPING(ENOTDIR), + MAPPING(EISDIR), + MAPPING(EINVAL), + MAPPING(ENFILE), + MAPPING(EMFILE), + MAPPING(ENOTTY), + MAPPING(ETXTBSY), + MAPPING(EFBIG), + MAPPING(ENOSPC), + MAPPING(ESPIPE), + MAPPING(EROFS), + MAPPING(EMLINK), + MAPPING(EPIPE), + MAPPING(EDOM), + MAPPING(ERANGE), + {-1, NULL} +}; + +static jclass enum_class; +static jmethodID enum_valueOf; +static jclass errno_class; + +void errno_enum_init(JNIEnv *env) { + if (enum_class != NULL) return; + + enum_class = (*env)->FindClass(env, "java/lang/Enum"); + PASS_EXCEPTIONS(env); + enum_class = (*env)->NewGlobalRef(env, enum_class); + enum_valueOf = (*env)->GetStaticMethodID(env, enum_class, + "valueOf", "(Ljava/lang/Class;Ljava/lang/String;)Ljava/lang/Enum;"); + PASS_EXCEPTIONS(env); + + errno_class = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/Errno"); + PASS_EXCEPTIONS(env); + errno_class = (*env)->NewGlobalRef(env, errno_class); +} + +void errno_enum_deinit(JNIEnv *env) { + if (enum_class != NULL) { + (*env)->DeleteGlobalRef(env, enum_class); + enum_class = NULL; + } + if (errno_class != NULL) { + (*env)->DeleteGlobalRef(env, errno_class); + errno_class = NULL; + } + enum_valueOf = NULL; +} + + +static char *errno_to_string(int errnum) { + int i; + for (i = 0; ERRNO_MAPPINGS[i].errno_str != NULL; i++) { + if (ERRNO_MAPPINGS[i].errno_val == errnum) + return ERRNO_MAPPINGS[i].errno_str; + } + return "UNKNOWN"; +} + +jobject errno_to_enum(JNIEnv *env, int errnum) { + char *str = errno_to_string(errnum); + assert(str != NULL); + + jstring jstr = (*env)->NewStringUTF(env, str); + PASS_EXCEPTIONS_RET(env, NULL); + + return (*env)->CallStaticObjectMethod( + env, enum_class, enum_valueOf, errno_class, jstr); +} Added: hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.h?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.h (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/errno_enum.h Fri Mar 4 04:43:43 2011 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ERRNO_ENUM_H +#define ERRNO_ENUM_H + +#include + +void errno_enum_init(JNIEnv *env); +void errno_enum_deinit(JNIEnv *env); +jobject errno_to_enum(JNIEnv *env, int errnum); + +#endif Added: hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c Fri Mar 4 04:43:43 2011 @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "file_descriptor.h" +#include "org_apache_hadoop.h" + +// class of java.io.FileDescriptor +static jclass fd_class; +// the internal field for the integer fd +static jfieldID fd_descriptor; +// the no-argument constructor +static jmethodID fd_constructor; + + +void fd_init(JNIEnv* env) +{ + if (fd_class != NULL) return; // already initted + + fd_class = (*env)->FindClass(env, "java/io/FileDescriptor"); + PASS_EXCEPTIONS(env); + fd_class = (*env)->NewGlobalRef(env, fd_class); + + fd_descriptor = (*env)->GetFieldID(env, fd_class, "fd", "I"); + PASS_EXCEPTIONS(env); + fd_constructor = (*env)->GetMethodID(env, fd_class, "", "()V"); +} + +void fd_deinit(JNIEnv *env) { + if (fd_class != NULL) { + (*env)->DeleteGlobalRef(env, fd_class); + fd_class = NULL; + } + fd_descriptor = NULL; + fd_constructor = NULL; +} + +/* + * Given an instance 'obj' of java.io.FileDescriptor, return the + * underlying fd, or throw if unavailable + */ +int fd_get(JNIEnv* env, jobject obj) { + return (*env)->GetIntField(env, obj, fd_descriptor); +} + +/* + * Create a FileDescriptor object corresponding to the given int fd + */ +jobject fd_create(JNIEnv *env, int fd) { + jobject obj = (*env)->NewObject(env, fd_class, fd_constructor); + PASS_EXCEPTIONS_RET(env, NULL); + + (*env)->SetIntField(env, obj, fd_descriptor, fd); + return obj; +} Added: hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h Fri Mar 4 04:43:43 2011 @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FILE_DESCRIPTOR_H +#define FILE_DESCRIPTOR_H + +#include + +void fd_init(JNIEnv *env); +void fd_deinit(JNIEnv *env); + +int fd_get(JNIEnv* env, jobject obj); +jobject fd_create(JNIEnv *env, int fd); + +#endif Modified: hadoop/common/branches/branch-0.20-security-patches/src/native/src/org_apache_hadoop.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/native/src/org_apache_hadoop.h?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/native/src/org_apache_hadoop.h (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/native/src/org_apache_hadoop.h Fri Mar 4 04:43:43 2011 @@ -50,6 +50,22 @@ } \ } +/* Helper macro to return if an exception is pending */ +#define PASS_EXCEPTIONS(env) \ + { \ + if ((*env)->ExceptionCheck(env)) return; \ + } + +#define PASS_EXCEPTIONS_GOTO(env, target) \ + { \ + if ((*env)->ExceptionCheck(env)) goto target; \ + } + +#define PASS_EXCEPTIONS_RET(env, ret) \ + { \ + if ((*env)->ExceptionCheck(env)) return (ret); \ + } + /** * A helper function to dlsym a 'symbol' from a given library-handle. * Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/TestSecureIOUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/TestSecureIOUtils.java?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/TestSecureIOUtils.java (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/TestSecureIOUtils.java Fri Mar 4 04:43:43 2011 @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.io.nativeio.NativeIO; + +import org.junit.BeforeClass; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assume.*; +import static org.junit.Assert.*; +import java.io.IOException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; + +public class TestSecureIOUtils { + private static String realOwner, realGroup; + private static final File testFilePath = + new File(System.getProperty("test.build.data"), "TestSecureIOContext"); + + @BeforeClass + public static void makeTestFile() throws Exception { + FileOutputStream fos = new FileOutputStream(testFilePath); + fos.write("hello".getBytes("UTF-8")); + fos.close(); + + Configuration conf = new Configuration(); + FileSystem rawFS = FileSystem.getLocal(conf).getRaw(); + FileStatus stat = rawFS.getFileStatus( + new Path(testFilePath.toString())); + realOwner = stat.getOwner(); + realGroup = stat.getGroup(); + } + + @Test + public void testReadUnrestricted() throws IOException { + SecureIOUtils.openForRead(testFilePath, null, null).close(); + } + + @Test + public void testReadCorrectlyRestrictedWithSecurity() throws IOException { + SecureIOUtils + .openForRead(testFilePath, realOwner, realGroup).close(); + } + + @Test + public void testReadIncorrectlyRestrictedWithSecurity() throws IOException { + try { + SecureIOUtils + .openForRead(testFilePath, "invalidUser", null).close(); + fail("Didn't throw expection for wrong ownership!"); + } catch (IOException ioe) { + // expected + } + } + + @Test + public void testCreateForWrite() throws IOException { + try { + SecureIOUtils.createForWrite(testFilePath, 0777); + fail("Was able to create file at " + testFilePath); + } catch (SecureIOUtils.AlreadyExistsException aee) { + // expected + } + } +} Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java?rev=1077680&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java (added) +++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/io/nativeio/TestNativeIO.java Fri Mar 4 04:43:43 2011 @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.nativeio; + +import java.io.File; +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.io.IOException; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assume.*; +import static org.junit.Assert.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.util.NativeCodeLoader; + +public class TestNativeIO { + static final Log LOG = LogFactory.getLog(TestNativeIO.class); + + static final File TEST_DIR = new File( + System.getProperty("test.build.data"), "testnativeio"); + + @Before + public void checkLoaded() { + assumeTrue(NativeCodeLoader.isNativeCodeLoaded()); + } + + @Before + public void setupTestDir() throws IOException { + FileUtil.fullyDelete(TEST_DIR); + TEST_DIR.mkdirs(); + } + + @Test + public void testFstat() throws Exception { + FileOutputStream fos = new FileOutputStream( + new File(TEST_DIR, "testfstat")); + NativeIO.Stat stat = NativeIO.fstat(fos.getFD()); + fos.close(); + LOG.info("Stat: " + String.valueOf(stat)); + + assertEquals(System.getProperty("user.name"), stat.getOwner()); + assertNotNull(stat.getGroup()); + assertTrue(!"".equals(stat.getGroup())); + assertEquals(NativeIO.Stat.S_IFREG, stat.getMode() & NativeIO.Stat.S_IFMT); + } + + @Test + public void testFstatClosedFd() throws Exception { + FileOutputStream fos = new FileOutputStream( + new File(TEST_DIR, "testfstat2")); + fos.close(); + try { + NativeIO.Stat stat = NativeIO.fstat(fos.getFD()); + } catch (IOException e) { + LOG.info("Got expected exception", e); + } + } + + @Test + public void testOpen() throws Exception { + LOG.info("Open a missing file without O_CREAT and it should fail"); + try { + FileDescriptor fd = NativeIO.open( + new File(TEST_DIR, "doesntexist").getAbsolutePath(), + NativeIO.O_WRONLY, 0700); + fail("Able to open a new file without O_CREAT"); + } catch (IOException ioe) { + // expected + } + + LOG.info("Test creating a file with O_CREAT"); + FileDescriptor fd = NativeIO.open( + new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(), + NativeIO.O_WRONLY | NativeIO.O_CREAT, 0700); + assertNotNull(true); + assertTrue(fd.valid()); + FileOutputStream fos = new FileOutputStream(fd); + fos.write("foo".getBytes()); + fos.close(); + + assertFalse(fd.valid()); + + LOG.info("Test exclusive create"); + try { + fd = NativeIO.open( + new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(), + NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL, 0700); + fail("Was able to create existing file with O_EXCL"); + } catch (IOException ioe) { + // expected + } + } + + /** + * Test that opens and closes a file 10000 times - this would crash with + * "Too many open files" if we leaked fds using this access pattern. + */ + @Test + public void testFDDoesntLeak() throws IOException { + for (int i = 0; i < 10000; i++) { + FileDescriptor fd = NativeIO.open( + new File(TEST_DIR, "testNoFdLeak").getAbsolutePath(), + NativeIO.O_WRONLY | NativeIO.O_CREAT, 0700); + assertNotNull(true); + assertTrue(fd.valid()); + FileOutputStream fos = new FileOutputStream(fd); + fos.write("foo".getBytes()); + fos.close(); + } + } + +} Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestIndexCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestIndexCache.java?rev=1077680&r1=1077679&r2=1077680&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestIndexCache.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestIndexCache.java Fri Mar 4 04:43:43 2011 @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.security.UserGroupInformation; import junit.framework.TestCase; @@ -55,7 +56,8 @@ public class TestIndexCache extends Test Path f = new Path(p, Integer.toString(totalsize, 36)); writeFile(fs, f, totalsize, partsPerMap); IndexRecord rec = cache.getIndexInformation( - Integer.toString(totalsize, 36), r.nextInt(partsPerMap), f); + Integer.toString(totalsize, 36), r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, totalsize); } @@ -66,7 +68,8 @@ public class TestIndexCache extends Test for (int i = bytesPerFile; i < 1024 * 1024; i += bytesPerFile) { Path f = new Path(p, Integer.toString(i, 36)); IndexRecord rec = cache.getIndexInformation(Integer.toString(i, 36), - r.nextInt(partsPerMap), f); + r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, i); } @@ -74,14 +77,16 @@ public class TestIndexCache extends Test Path f = new Path(p, Integer.toString(totalsize, 36)); writeFile(fs, f, totalsize, partsPerMap); cache.getIndexInformation(Integer.toString(totalsize, 36), - r.nextInt(partsPerMap), f); + r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); fs.delete(f, false); // oldest fails to read, or error boolean fnf = false; try { cache.getIndexInformation(Integer.toString(bytesPerFile, 36), - r.nextInt(partsPerMap), new Path(p, Integer.toString(bytesPerFile))); + r.nextInt(partsPerMap), new Path(p, Integer.toString(bytesPerFile)), + UserGroupInformation.getCurrentUser().getShortUserName()); } catch (IOException e) { if (e.getCause() == null || !(e.getCause() instanceof FileNotFoundException)) { @@ -96,11 +101,13 @@ public class TestIndexCache extends Test // should find all the other entries for (int i = bytesPerFile << 1; i < 1024 * 1024; i += bytesPerFile) { IndexRecord rec = cache.getIndexInformation(Integer.toString(i, 36), - r.nextInt(partsPerMap), new Path(p, Integer.toString(i, 36))); + r.nextInt(partsPerMap), new Path(p, Integer.toString(i, 36)), + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, i); } IndexRecord rec = cache.getIndexInformation(Integer.toString(totalsize, 36), - r.nextInt(partsPerMap), f); + r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, totalsize); } @@ -130,7 +137,8 @@ public class TestIndexCache extends Test out.writeLong(iout.getChecksum().getValue()); dout.close(); try { - cache.getIndexInformation("badindex", 7, f); + cache.getIndexInformation("badindex", 7, f, + UserGroupInformation.getCurrentUser().getShortUserName()); fail("Did not detect bad checksum"); } catch (IOException e) { if (!(e.getCause() instanceof ChecksumException)) {