Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4422E1142F for ; Wed, 18 Jun 2014 05:10:32 +0000 (UTC) Received: (qmail 96912 invoked by uid 500); 18 Jun 2014 05:10:32 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 96858 invoked by uid 500); 18 Jun 2014 05:10:32 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 96846 invoked by uid 99); 18 Jun 2014 05:10:32 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Jun 2014 05:10:32 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Jun 2014 05:10:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 5360D2388ADA; Wed, 18 Jun 2014 05:09:36 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1603355 [2/3] - in /hadoop/common/branches/HDFS-5442/hadoop-hdfs-project: hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/ hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop-hdfs-httpfs/src/test/jav... Date: Wed, 18 Jun 2014 05:09:34 -0000 To: hdfs-commits@hadoop.apache.org From: vinayakumarb@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140618050936.5360D2388ADA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Wed Jun 18 05:09:28 2014 @@ -50,7 +50,6 @@ import org.apache.hadoop.fs.permission.P import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.AclException; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -265,11 +264,6 @@ public class FSDirectory implements Clos ready = flag; } - private void incrDeletedFileCount(long count) { - if (getFSNamesystem() != null) - NameNode.getNameNodeMetrics().incrFilesDeleted(count); - } - /** * Shutdown the filestore */ @@ -321,19 +315,7 @@ public class FSDirectory implements Clos UnresolvedLinkException, SnapshotAccessControlException, AclException { waitForReady(); - // Always do an implicit mkdirs for parent directory tree. long modTime = now(); - - Path parent = new Path(path).getParent(); - if (parent == null) { - // Trying to add "/" as a file - this path has no - // parent -- avoids an NPE below. - return null; - } - - if (!mkdirs(parent.toString(), permissions, true, modTime)) { - return null; - } INodeFile newNode = new INodeFile(namesystem.allocateNewInodeId(), null, permissions, modTime, modTime, BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize); @@ -437,65 +419,6 @@ public class FSDirectory implements Clos } /** - * Persist the block list for the inode. - */ - void persistBlocks(String path, INodeFile file, boolean logRetryCache) { - Preconditions.checkArgument(file.isUnderConstruction()); - waitForReady(); - - writeLock(); - try { - fsImage.getEditLog().logUpdateBlocks(path, file, logRetryCache); - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* FSDirectory.persistBlocks: " - +path+" with "+ file.getBlocks().length - +" blocks is persisted to the file system"); - } - } finally { - writeUnlock(); - } - } - - /** - * Persist the new block (the last block of the given file). - */ - void persistNewBlock(String path, INodeFile file) { - Preconditions.checkArgument(file.isUnderConstruction()); - waitForReady(); - - writeLock(); - try { - fsImage.getEditLog().logAddBlock(path, file); - } finally { - writeUnlock(); - } - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* FSDirectory.persistNewBlock: " - + path + " with new block " + file.getLastBlock().toString() - + ", current total block count is " + file.getBlocks().length); - } - } - - /** - * Close file. - */ - void closeFile(String path, INodeFile file) { - waitForReady(); - writeLock(); - try { - // file is closed - fsImage.getEditLog().logCloseFile(path, file); - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* FSDirectory.closeFile: " - +path+" with "+ file.getBlocks().length - +" blocks is persisted to the file system"); - } - } finally { - writeUnlock(); - } - } - - /** * Remove a block from the file. * @return Whether the block exists in the corresponding file */ @@ -540,7 +463,7 @@ public class FSDirectory implements Clos * @deprecated Use {@link #renameTo(String, String, boolean, Rename...)} */ @Deprecated - boolean renameTo(String src, String dst, boolean logRetryCache) + boolean renameTo(String src, String dst, long mtime) throws QuotaExceededException, UnresolvedLinkException, FileAlreadyExistsException, SnapshotAccessControlException, IOException { if (NameNode.stateChangeLog.isDebugEnabled()) { @@ -548,22 +471,20 @@ public class FSDirectory implements Clos +src+" to "+dst); } waitForReady(); - long now = now(); writeLock(); try { - if (!unprotectedRenameTo(src, dst, now)) + if (!unprotectedRenameTo(src, dst, mtime)) return false; } finally { writeUnlock(); } - fsImage.getEditLog().logRename(src, dst, now, logRetryCache); return true; } /** * @see #unprotectedRenameTo(String, String, long, Options.Rename...) */ - void renameTo(String src, String dst, boolean logRetryCache, + void renameTo(String src, String dst, long mtime, Options.Rename... options) throws FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, QuotaExceededException, @@ -573,16 +494,14 @@ public class FSDirectory implements Clos + " to " + dst); } waitForReady(); - long now = now(); writeLock(); try { - if (unprotectedRenameTo(src, dst, now, options)) { - incrDeletedFileCount(1); + if (unprotectedRenameTo(src, dst, mtime, options)) { + namesystem.incrDeletedFileCount(1); } } finally { writeUnlock(); } - fsImage.getEditLog().logRename(src, dst, now, logRetryCache, options); } /** @@ -1106,11 +1025,7 @@ public class FSDirectory implements Clos waitForReady(); writeLock(); try { - final Block[] fileBlocks = unprotectedSetReplication( - src, replication, blockRepls); - if (fileBlocks != null) // log replication change - fsImage.getEditLog().logSetReplication(src, replication); - return fileBlocks; + return unprotectedSetReplication(src, replication, blockRepls); } finally { writeUnlock(); } @@ -1178,7 +1093,6 @@ public class FSDirectory implements Clos } finally { writeUnlock(); } - fsImage.getEditLog().logSetPermissions(src, permission); } void unprotectedSetPermission(String src, FsPermission permissions) @@ -1203,7 +1117,6 @@ public class FSDirectory implements Clos } finally { writeUnlock(); } - fsImage.getEditLog().logSetOwner(src, username, groupname); } void unprotectedSetOwner(String src, String username, String groupname) @@ -1226,18 +1139,14 @@ public class FSDirectory implements Clos /** * Concat all the blocks from srcs to trg and delete the srcs files */ - void concat(String target, String [] srcs, boolean supportRetryCache) + void concat(String target, String[] srcs, long timestamp) throws UnresolvedLinkException, QuotaExceededException, SnapshotAccessControlException, SnapshotException { writeLock(); try { // actual move waitForReady(); - long timestamp = now(); unprotectedConcat(target, srcs, timestamp); - // do the commit - fsImage.getEditLog().logConcat(target, srcs, timestamp, - supportRetryCache); } finally { writeUnlock(); } @@ -1312,17 +1221,14 @@ public class FSDirectory implements Clos * @param src Path of a directory to delete * @param collectedBlocks Blocks under the deleted directory * @param removedINodes INodes that should be removed from {@link #inodeMap} - * @param logRetryCache Whether to record RPC IDs in editlog to support retry - * cache rebuilding. - * @return true on successful deletion; else false + * @return the number of files that have been removed */ - boolean delete(String src, BlocksMapUpdateInfo collectedBlocks, - List removedINodes, boolean logRetryCache) throws IOException { + long delete(String src, BlocksMapUpdateInfo collectedBlocks, + List removedINodes, long mtime) throws IOException { if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src); } waitForReady(); - long now = now(); final long filesRemoved; writeLock(); try { @@ -1335,20 +1241,13 @@ public class FSDirectory implements Clos new ArrayList(); checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs); filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks, - removedINodes, now); + removedINodes, mtime); namesystem.removeSnapshottableDirs(snapshottableDirs); } } finally { writeUnlock(); } - if (filesRemoved < 0) { - return false; - } - fsImage.getEditLog().logDelete(src, now, logRetryCache); - incrDeletedFileCount(filesRemoved); - // Blocks/INodes will be handled later by the caller of this method - getFSNamesystem().removePathAndBlocks(src, null, null); - return true; + return filesRemoved; } private static boolean deleteAllowed(final INodesInPath iip, @@ -1895,112 +1794,6 @@ public class FSDirectory implements Clos // inodes can be null only when its called without holding lock return inodes == null ? "" : getFullPathName(inodes, inodes.length - 1); } - - /** - * Create a directory - * If ancestor directories do not exist, automatically create them. - - * @param src string representation of the path to the directory - * @param permissions the permission of the directory - * @param inheritPermission if the permission of the directory should inherit - * from its parent or not. u+wx is implicitly added to - * the automatically created directories, and to the - * given directory if inheritPermission is true - * @param now creation time - * @return true if the operation succeeds false otherwise - * @throws QuotaExceededException if directory creation violates - * any quota limit - * @throws UnresolvedLinkException if a symlink is encountered in src. - * @throws SnapshotAccessControlException if path is in RO snapshot - */ - boolean mkdirs(String src, PermissionStatus permissions, - boolean inheritPermission, long now) - throws FileAlreadyExistsException, QuotaExceededException, - UnresolvedLinkException, SnapshotAccessControlException, - AclException { - src = normalizePath(src); - String[] names = INode.getPathNames(src); - byte[][] components = INode.getPathComponents(names); - final int lastInodeIndex = components.length - 1; - - writeLock(); - try { - INodesInPath iip = getExistingPathINodes(components); - if (iip.isSnapshot()) { - throw new SnapshotAccessControlException( - "Modification on RO snapshot is disallowed"); - } - INode[] inodes = iip.getINodes(); - - // find the index of the first null in inodes[] - StringBuilder pathbuilder = new StringBuilder(); - int i = 1; - for(; i < inodes.length && inodes[i] != null; i++) { - pathbuilder.append(Path.SEPARATOR).append(names[i]); - if (!inodes[i].isDirectory()) { - throw new FileAlreadyExistsException("Parent path is not a directory: " - + pathbuilder+ " "+inodes[i].getLocalName()); - } - } - - // default to creating parent dirs with the given perms - PermissionStatus parentPermissions = permissions; - - // if not inheriting and it's the last inode, there's no use in - // computing perms that won't be used - if (inheritPermission || (i < lastInodeIndex)) { - // if inheriting (ie. creating a file or symlink), use the parent dir, - // else the supplied permissions - // NOTE: the permissions of the auto-created directories violate posix - FsPermission parentFsPerm = inheritPermission - ? inodes[i-1].getFsPermission() : permissions.getPermission(); - - // ensure that the permissions allow user write+execute - if (!parentFsPerm.getUserAction().implies(FsAction.WRITE_EXECUTE)) { - parentFsPerm = new FsPermission( - parentFsPerm.getUserAction().or(FsAction.WRITE_EXECUTE), - parentFsPerm.getGroupAction(), - parentFsPerm.getOtherAction() - ); - } - - if (!parentPermissions.getPermission().equals(parentFsPerm)) { - parentPermissions = new PermissionStatus( - parentPermissions.getUserName(), - parentPermissions.getGroupName(), - parentFsPerm - ); - // when inheriting, use same perms for entire path - if (inheritPermission) permissions = parentPermissions; - } - } - - // create directories beginning from the first null index - for(; i < inodes.length; i++) { - pathbuilder.append(Path.SEPARATOR).append(names[i]); - unprotectedMkdir(namesystem.allocateNewInodeId(), iip, i, - components[i], (i < lastInodeIndex) ? parentPermissions - : permissions, null, now); - if (inodes[i] == null) { - return false; - } - // Directory creation also count towards FilesCreated - // to match count of FilesDeleted metric. - if (getFSNamesystem() != null) - NameNode.getNameNodeMetrics().incrFilesCreated(); - - final String cur = pathbuilder.toString(); - fsImage.getEditLog().logMkDir(cur, inodes[i]); - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( - "DIR* FSDirectory.mkdirs: created directory " + cur); - } - } - } finally { - writeUnlock(); - } - return true; - } INode unprotectedMkdir(long inodeId, String src, PermissionStatus permissions, List aclEntries, long timestamp) @@ -2019,7 +1812,7 @@ public class FSDirectory implements Clos * The parent path to the directory is at [0, pos-1]. * All ancestors exist. Newly created one stored at index pos. */ - private void unprotectedMkdir(long inodeId, INodesInPath inodesInPath, + void unprotectedMkdir(long inodeId, INodesInPath inodesInPath, int pos, byte[] name, PermissionStatus permission, List aclEntries, long timestamp) throws QuotaExceededException, AclException { @@ -2331,10 +2124,8 @@ public class FSDirectory implements Clos } return 1; } - - /** - */ - String normalizePath(String src) { + + static String normalizePath(String src) { if (src.length() > 1 && src.endsWith("/")) { src = src.substring(0, src.length() - 1); } @@ -2419,7 +2210,7 @@ public class FSDirectory implements Clos /** * See {@link ClientProtocol#setQuota(String, long, long)} for the contract. * Sets quota for for a directory. - * @return INodeDirectory if any of the quotas have changed. null other wise. + * @return INodeDirectory if any of the quotas have changed. null otherwise. * @throws FileNotFoundException if the path does not exist. * @throws PathIsNotDirectoryException if the path is not a directory. * @throws QuotaExceededException if the directory tree size is @@ -2470,21 +2261,17 @@ public class FSDirectory implements Clos /** * See {@link ClientProtocol#setQuota(String, long, long)} for the contract. + * @return INodeDirectory if any of the quotas have changed. null otherwise. * @throws SnapshotAccessControlException if path is in RO snapshot * @see #unprotectedSetQuota(String, long, long) */ - void setQuota(String src, long nsQuota, long dsQuota) + INodeDirectory setQuota(String src, long nsQuota, long dsQuota) throws FileNotFoundException, PathIsNotDirectoryException, QuotaExceededException, UnresolvedLinkException, SnapshotAccessControlException { writeLock(); try { - INodeDirectory dir = unprotectedSetQuota(src, nsQuota, dsQuota); - if (dir != null) { - final Quota.Counts q = dir.getQuotaCounts(); - fsImage.getEditLog().logSetQuota(src, - q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE)); - } + return unprotectedSetQuota(src, nsQuota, dsQuota); } finally { writeUnlock(); } @@ -2503,18 +2290,14 @@ public class FSDirectory implements Clos /** * Sets the access time on the file/directory. Logs it in the transaction log. */ - void setTimes(String src, INode inode, long mtime, long atime, boolean force, - int latestSnapshotId) throws QuotaExceededException { - boolean status = false; + boolean setTimes(INode inode, long mtime, long atime, boolean force, + int latestSnapshotId) throws QuotaExceededException { writeLock(); try { - status = unprotectedSetTimes(inode, mtime, atime, force, latestSnapshotId); + return unprotectedSetTimes(inode, mtime, atime, force, latestSnapshotId); } finally { writeUnlock(); } - if (status) { - fsImage.getEditLog().logTimes(src, mtime, atime); - } } boolean unprotectedSetTimes(String src, long mtime, long atime, boolean force) @@ -2678,49 +2461,21 @@ public class FSDirectory implements Clos } return perm; } - + /** - * Add the given symbolic link to the fs. Record it in the edits log. + * Add the specified path into the namespace. */ - INodeSymlink addSymlink(String path, String target, - PermissionStatus dirPerms, boolean createParent, boolean logRetryCache) - throws UnresolvedLinkException, FileAlreadyExistsException, - QuotaExceededException, SnapshotAccessControlException, AclException { - waitForReady(); - - final long modTime = now(); - if (createParent) { - final String parent = new Path(path).getParent().toString(); - if (!mkdirs(parent, dirPerms, true, modTime)) { - return null; - } - } - final String userName = dirPerms.getUserName(); - INodeSymlink newNode = null; - long id = namesystem.allocateNewInodeId(); + INodeSymlink addSymlink(long id, String path, String target, + long mtime, long atime, PermissionStatus perm) + throws UnresolvedLinkException, QuotaExceededException { writeLock(); try { - newNode = unprotectedAddSymlink(id, path, target, modTime, modTime, - new PermissionStatus(userName, null, FsPermission.getDefault())); + return unprotectedAddSymlink(id, path, target, mtime, atime, perm); } finally { writeUnlock(); } - if (newNode == null) { - NameNode.stateChangeLog.info("DIR* addSymlink: failed to add " + path); - return null; - } - fsImage.getEditLog().logSymlink(path, target, modTime, modTime, newNode, - logRetryCache); - - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* addSymlink: " + path + " is added"); - } - return newNode; } - /** - * Add the specified path into the namespace. Invoked from edit log processing. - */ INodeSymlink unprotectedAddSymlink(long id, String path, String target, long mtime, long atime, PermissionStatus perm) throws UnresolvedLinkException, QuotaExceededException { @@ -2730,11 +2485,10 @@ public class FSDirectory implements Clos return addINode(path, symlink) ? symlink : null; } - void modifyAclEntries(String src, List aclSpec) throws IOException { + List modifyAclEntries(String src, List aclSpec) throws IOException { writeLock(); try { - List newAcl = unprotectedModifyAclEntries(src, aclSpec); - fsImage.getEditLog().logSetAcl(src, newAcl); + return unprotectedModifyAclEntries(src, aclSpec); } finally { writeUnlock(); } @@ -2753,11 +2507,10 @@ public class FSDirectory implements Clos return newAcl; } - void removeAclEntries(String src, List aclSpec) throws IOException { + List removeAclEntries(String src, List aclSpec) throws IOException { writeLock(); try { - List newAcl = unprotectedRemoveAclEntries(src, aclSpec); - fsImage.getEditLog().logSetAcl(src, newAcl); + return unprotectedRemoveAclEntries(src, aclSpec); } finally { writeUnlock(); } @@ -2776,11 +2529,10 @@ public class FSDirectory implements Clos return newAcl; } - void removeDefaultAcl(String src) throws IOException { + List removeDefaultAcl(String src) throws IOException { writeLock(); try { - List newAcl = unprotectedRemoveDefaultAcl(src); - fsImage.getEditLog().logSetAcl(src, newAcl); + return unprotectedRemoveDefaultAcl(src); } finally { writeUnlock(); } @@ -2803,7 +2555,6 @@ public class FSDirectory implements Clos writeLock(); try { unprotectedRemoveAcl(src); - fsImage.getEditLog().logSetAcl(src, AclFeature.EMPTY_ENTRY_LIST); } finally { writeUnlock(); } @@ -2817,11 +2568,10 @@ public class FSDirectory implements Clos AclStorage.removeINodeAcl(inode, snapshotId); } - void setAcl(String src, List aclSpec) throws IOException { + List setAcl(String src, List aclSpec) throws IOException { writeLock(); try { - List newAcl = unprotectedSetAcl(src, aclSpec); - fsImage.getEditLog().logSetAcl(src, newAcl); + return unprotectedSetAcl(src, aclSpec); } finally { writeUnlock(); } @@ -2868,18 +2618,11 @@ public class FSDirectory implements Clos readUnlock(); } } - - void removeXAttr(String src, XAttr xAttr) throws IOException { + + XAttr removeXAttr(String src, XAttr xAttr) throws IOException { writeLock(); try { - XAttr removedXAttr = unprotectedRemoveXAttr(src, xAttr); - if (removedXAttr != null) { - fsImage.getEditLog().logRemoveXAttr(src, removedXAttr); - } else { - NameNode.stateChangeLog.info("DIR* FSDirectory.removeXAttr: XAttr " + - XAttrHelper.getPrefixName(xAttr) + - " does not exist on the path " + src); - } + return unprotectedRemoveXAttr(src, xAttr); } finally { writeUnlock(); } @@ -2917,12 +2660,11 @@ public class FSDirectory implements Clos return xAttrs; } - void setXAttr(String src, XAttr xAttr, EnumSet flag, - boolean logRetryCache) throws IOException { + void setXAttr(String src, XAttr xAttr, EnumSet flag) + throws IOException { writeLock(); try { unprotectedSetXAttr(src, xAttr, flag); - fsImage.getEditLog().logSetXAttr(src, xAttr, logRetryCache); } finally { writeUnlock(); } @@ -2943,6 +2685,7 @@ public class FSDirectory implements Clos EnumSet flag) throws QuotaExceededException, IOException { List xAttrs = Lists.newArrayListWithCapacity( existingXAttrs != null ? existingXAttrs.size() + 1 : 1); + int userVisibleXAttrsNum = 0; // Number of user visible xAttrs boolean exist = false; if (existingXAttrs != null) { for (XAttr a: existingXAttrs) { @@ -2951,6 +2694,10 @@ public class FSDirectory implements Clos exist = true; } else { xAttrs.add(a); + + if (isUserVisible(a)) { + userVisibleXAttrsNum++; + } } } } @@ -2958,7 +2705,11 @@ public class FSDirectory implements Clos XAttrSetFlag.validate(xAttr.getName(), exist, flag); xAttrs.add(xAttr); - if (xAttrs.size() > inodeXAttrsLimit) { + if (isUserVisible(xAttr)) { + userVisibleXAttrsNum++; + } + + if (userVisibleXAttrsNum > inodeXAttrsLimit) { throw new IOException("Cannot add additional XAttr to inode, " + "would exceed limit of " + inodeXAttrsLimit); } @@ -2966,6 +2717,14 @@ public class FSDirectory implements Clos return xAttrs; } + private boolean isUserVisible(XAttr xAttr) { + if (xAttr.getNameSpace() == XAttr.NameSpace.USER || + xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED) { + return true; + } + return false; + } + List getXAttrs(String src) throws IOException { String srcs = normalizePath(src); readLock(); Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Jun 18 05:09:28 2014 @@ -145,7 +145,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.StorageType; -import org.apache.hadoop.hdfs.XAttrHelper; +import org.apache.hadoop.hdfs.protocol.AclException; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; @@ -167,6 +167,7 @@ import org.apache.hadoop.hdfs.protocol.Q import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.protocol.RollingUpgradeException; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; +import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; @@ -1567,6 +1568,7 @@ public class FSNamesystem implements Nam src = FSDirectory.resolvePath(src, pathComponents, dir); checkOwner(pc, src); dir.setPermission(src, permission); + getEditLog().logSetPermissions(src, permission); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -1612,6 +1614,7 @@ public class FSNamesystem implements Nam } } dir.setOwner(src, username, group); + getEditLog().logSetOwner(src, username, group); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -1742,7 +1745,11 @@ public class FSNamesystem implements Nam if (isReadOp) { continue; } - dir.setTimes(src, inode, -1, now, false, iip.getLatestSnapshotId()); + boolean changed = dir.setTimes(inode, -1, now, false, + iip.getLatestSnapshotId()); + if (changed) { + getEditLog().logTimes(src, -1, now); + } } } final long fileSize = iip.isSnapshot() ? @@ -1953,7 +1960,9 @@ public class FSNamesystem implements Nam Arrays.toString(srcs) + " to " + target); } - dir.concat(target,srcs, logRetryCache); + long timestamp = now(); + dir.concat(target, srcs, timestamp); + getEditLog().logConcat(target, srcs, timestamp, logRetryCache); } /** @@ -1994,7 +2003,11 @@ public class FSNamesystem implements Nam final INodesInPath iip = dir.getINodesInPath4Write(src); final INode inode = iip.getLastINode(); if (inode != null) { - dir.setTimes(src, inode, mtime, atime, true, iip.getLatestSnapshotId()); + boolean changed = dir.setTimes(inode, mtime, atime, true, + iip.getLatestSnapshotId()); + if (changed) { + getEditLog().logTimes(src, mtime, atime); + } resultingStat = getAuditFileInfo(src, false); } else { throw new FileNotFoundException("File/Directory " + src + " does not exist."); @@ -2063,7 +2076,7 @@ public class FSNamesystem implements Nam checkFsObjectLimit(); // add symbolic link to namespace - dir.addSymlink(link, target, dirPerms, createParent, logRetryCache); + addSymlink(link, target, dirPerms, createParent, logRetryCache); resultingStat = getAuditFileInfo(link, false); } finally { writeUnlock(); @@ -2115,6 +2128,7 @@ public class FSNamesystem implements Nam final Block[] blocks = dir.setReplication(src, replication, blockRepls); isFile = blocks != null; if (isFile) { + getEditLog().logSetReplication(src, replication); blockManager.setReplication(blockRepls[0], blockRepls[1], src, blocks); } } finally { @@ -2315,8 +2329,16 @@ public class FSNamesystem implements Nam final DatanodeDescriptor clientNode = blockManager.getDatanodeManager().getDatanodeByHost(clientMachine); - INodeFile newNode = dir.addFile(src, permissions, replication, blockSize, - holder, clientMachine, clientNode); + INodeFile newNode = null; + + // Always do an implicit mkdirs for parent directory tree. + Path parent = new Path(src).getParent(); + if (parent != null && mkdirsRecursively(parent.toString(), + permissions, true, now())) { + newNode = dir.addFile(src, permissions, replication, blockSize, + holder, clientMachine, clientNode); + } + if (newNode == null) { throw new IOException("Unable to add " + src + " to namespace"); } @@ -2740,7 +2762,7 @@ public class FSNamesystem implements Nam INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile); saveAllocatedBlock(src, inodesInPath, newBlock, targets); - dir.persistNewBlock(src, pendingFile); + persistNewBlock(src, pendingFile); offset = pendingFile.computeFileSize(); } finally { writeUnlock(); @@ -2960,7 +2982,7 @@ public class FSNamesystem implements Nam NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: " + b + " is removed from pendingCreates"); } - dir.persistBlocks(src, file, false); + persistBlocks(src, file, false); } finally { writeUnlock(); } @@ -2996,6 +3018,13 @@ public class FSNamesystem implements Nam + (lease != null ? lease.toString() : "Holder " + holder + " does not have any open files.")); } + // No further modification is allowed on a deleted file. + // A file is considered deleted, if it has no parent or is marked + // as deleted in the snapshot feature. + if (file.getParent() == null || (file.isWithSnapshot() && + file.getFileWithSnapshotFeature().isCurrentFileDeleted())) { + throw new FileNotFoundException(src); + } String clientName = file.getFileUnderConstructionFeature().getClientName(); if (holder != null && !clientName.equals(holder)) { throw new LeaseExpiredException("Lease mismatch on " + ident + @@ -3260,7 +3289,9 @@ public class FSNamesystem implements Nam false, false); } - if (dir.renameTo(src, dst, logRetryCache)) { + long mtime = now(); + if (dir.renameTo(src, dst, mtime)) { + getEditLog().logRename(src, dst, mtime, logRetryCache); return true; } return false; @@ -3325,7 +3356,9 @@ public class FSNamesystem implements Nam false); } - dir.renameTo(src, dst, logRetryCache, options); + long mtime = now(); + dir.renameTo(src, dst, mtime, options); + getEditLog().logRename(src, dst, mtime, logRetryCache, options); } /** @@ -3408,10 +3441,17 @@ public class FSNamesystem implements Nam checkPermission(pc, src, false, null, FsAction.WRITE, null, FsAction.ALL, true, false); } + long mtime = now(); // Unlink the target directory from directory tree - if (!dir.delete(src, collectedBlocks, removedINodes, logRetryCache)) { + long filesRemoved = dir.delete(src, collectedBlocks, removedINodes, + mtime); + if (filesRemoved < 0) { return false; } + getEditLog().logDelete(src, mtime, logRetryCache); + incrDeletedFileCount(filesRemoved); + // Blocks/INodes will be handled later + removePathAndBlocks(src, null, null); ret = true; } finally { writeUnlock(); @@ -3419,6 +3459,7 @@ public class FSNamesystem implements Nam getEditLog().logSync(); removeBlocks(collectedBlocks); // Incremental deletion of blocks collectedBlocks.clear(); + dir.writeLock(); try { dir.removeFromInodeMap(removedINodes); @@ -3671,13 +3712,119 @@ public class FSNamesystem implements Nam // create multiple inodes. checkFsObjectLimit(); - if (!dir.mkdirs(src, permissions, false, now())) { + if (!mkdirsRecursively(src, permissions, false, now())) { throw new IOException("Failed to create directory: " + src); } return true; } /** + * Create a directory + * If ancestor directories do not exist, automatically create them. + + * @param src string representation of the path to the directory + * @param permissions the permission of the directory + * @param inheritPermission if the permission of the directory should inherit + * from its parent or not. u+wx is implicitly added to + * the automatically created directories, and to the + * given directory if inheritPermission is true + * @param now creation time + * @return true if the operation succeeds false otherwise + * @throws QuotaExceededException if directory creation violates + * any quota limit + * @throws UnresolvedLinkException if a symlink is encountered in src. + * @throws SnapshotAccessControlException if path is in RO snapshot + */ + private boolean mkdirsRecursively(String src, PermissionStatus permissions, + boolean inheritPermission, long now) + throws FileAlreadyExistsException, QuotaExceededException, + UnresolvedLinkException, SnapshotAccessControlException, + AclException { + src = FSDirectory.normalizePath(src); + String[] names = INode.getPathNames(src); + byte[][] components = INode.getPathComponents(names); + final int lastInodeIndex = components.length - 1; + + dir.writeLock(); + try { + INodesInPath iip = dir.getExistingPathINodes(components); + if (iip.isSnapshot()) { + throw new SnapshotAccessControlException( + "Modification on RO snapshot is disallowed"); + } + INode[] inodes = iip.getINodes(); + + // find the index of the first null in inodes[] + StringBuilder pathbuilder = new StringBuilder(); + int i = 1; + for(; i < inodes.length && inodes[i] != null; i++) { + pathbuilder.append(Path.SEPARATOR).append(names[i]); + if (!inodes[i].isDirectory()) { + throw new FileAlreadyExistsException( + "Parent path is not a directory: " + + pathbuilder + " "+inodes[i].getLocalName()); + } + } + + // default to creating parent dirs with the given perms + PermissionStatus parentPermissions = permissions; + + // if not inheriting and it's the last inode, there's no use in + // computing perms that won't be used + if (inheritPermission || (i < lastInodeIndex)) { + // if inheriting (ie. creating a file or symlink), use the parent dir, + // else the supplied permissions + // NOTE: the permissions of the auto-created directories violate posix + FsPermission parentFsPerm = inheritPermission + ? inodes[i-1].getFsPermission() : permissions.getPermission(); + + // ensure that the permissions allow user write+execute + if (!parentFsPerm.getUserAction().implies(FsAction.WRITE_EXECUTE)) { + parentFsPerm = new FsPermission( + parentFsPerm.getUserAction().or(FsAction.WRITE_EXECUTE), + parentFsPerm.getGroupAction(), + parentFsPerm.getOtherAction() + ); + } + + if (!parentPermissions.getPermission().equals(parentFsPerm)) { + parentPermissions = new PermissionStatus( + parentPermissions.getUserName(), + parentPermissions.getGroupName(), + parentFsPerm + ); + // when inheriting, use same perms for entire path + if (inheritPermission) permissions = parentPermissions; + } + } + + // create directories beginning from the first null index + for(; i < inodes.length; i++) { + pathbuilder.append(Path.SEPARATOR).append(names[i]); + dir.unprotectedMkdir(allocateNewInodeId(), iip, i, components[i], + (i < lastInodeIndex) ? parentPermissions : permissions, null, + now); + if (inodes[i] == null) { + return false; + } + // Directory creation also count towards FilesCreated + // to match count of FilesDeleted metric. + NameNode.getNameNodeMetrics().incrFilesCreated(); + + final String cur = pathbuilder.toString(); + getEditLog().logMkDir(cur, inodes[i]); + if(NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug( + "mkdirs: created directory " + cur); + } + } + } finally { + dir.writeUnlock(); + } + return true; + } + + /** * Get the content summary for a specific file/dir. * * @param src The string representation of the path to the file @@ -3721,7 +3868,7 @@ public class FSNamesystem implements Nam * * Note: This does not support ".inodes" relative path. */ - void setQuota(String path, long nsQuota, long dsQuota) + void setQuota(String path, long nsQuota, long dsQuota) throws IOException, UnresolvedLinkException { checkSuperuserPrivilege(); checkOperation(OperationCategory.WRITE); @@ -3729,7 +3876,12 @@ public class FSNamesystem implements Nam try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set quota on " + path); - dir.setQuota(path, nsQuota, dsQuota); + INodeDirectory changed = dir.setQuota(path, nsQuota, dsQuota); + if (changed != null) { + final Quota.Counts q = changed.getQuotaCounts(); + getEditLog().logSetQuota(path, + q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE)); + } } finally { writeUnlock(); } @@ -3770,7 +3922,7 @@ public class FSNamesystem implements Nam pendingFile.getFileUnderConstructionFeature().updateLengthOfLastBlock( pendingFile, lastBlockLength); } - dir.persistBlocks(src, pendingFile, false); + persistBlocks(src, pendingFile, false); } finally { writeUnlock(); } @@ -3963,7 +4115,7 @@ public class FSNamesystem implements Nam final INodeFile newFile = pendingFile.toCompleteFile(now()); // close file and persist block allocations for this file - dir.closeFile(src, newFile); + closeFile(src, newFile); blockManager.checkReplication(newFile); } @@ -4114,7 +4266,8 @@ public class FSNamesystem implements Nam src = closeFileCommitBlocks(iFile, storedBlock); } else { // If this commit does not want to close the file, persist blocks - src = persistBlocks(iFile, false); + src = iFile.getFullPathName(); + persistBlocks(src, iFile, false); } } finally { writeUnlock(); @@ -4153,21 +4306,6 @@ public class FSNamesystem implements Nam } /** - * Persist the block list for the given file. - * - * @param pendingFile - * @return Path to the given file. - * @throws IOException - */ - @VisibleForTesting - String persistBlocks(INodeFile pendingFile, boolean logRetryCache) - throws IOException { - String src = pendingFile.getFullPathName(); - dir.persistBlocks(src, pendingFile, logRetryCache); - return src; - } - - /** * Renew the lease(s) held by the given client */ void renewLease(String holder) throws IOException { @@ -4351,6 +4489,85 @@ public class FSNamesystem implements Nam } /** + * Persist the block list for the inode. + * @param path + * @param file + * @param logRetryCache + */ + private void persistBlocks(String path, INodeFile file, + boolean logRetryCache) { + assert hasWriteLock(); + Preconditions.checkArgument(file.isUnderConstruction()); + getEditLog().logUpdateBlocks(path, file, logRetryCache); + if(NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug("persistBlocks: " + path + + " with " + file.getBlocks().length + " blocks is persisted to" + + " the file system"); + } + } + + void incrDeletedFileCount(long count) { + NameNode.getNameNodeMetrics().incrFilesDeleted(count); + } + + /** + * Close file. + * @param path + * @param file + */ + private void closeFile(String path, INodeFile file) { + assert hasWriteLock(); + dir.waitForReady(); + // file is closed + getEditLog().logCloseFile(path, file); + if (NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug("closeFile: " + +path+" with "+ file.getBlocks().length + +" blocks is persisted to the file system"); + } + } + + /** + * Add the given symbolic link to the fs. Record it in the edits log. + * @param path + * @param target + * @param dirPerms + * @param createParent + * @param logRetryCache + * @param dir + */ + private INodeSymlink addSymlink(String path, String target, + PermissionStatus dirPerms, + boolean createParent, boolean logRetryCache) + throws UnresolvedLinkException, FileAlreadyExistsException, + QuotaExceededException, SnapshotAccessControlException, AclException { + dir.waitForReady(); + + final long modTime = now(); + if (createParent) { + final String parent = new Path(path).getParent().toString(); + if (!mkdirsRecursively(parent, dirPerms, true, modTime)) { + return null; + } + } + final String userName = dirPerms.getUserName(); + long id = allocateNewInodeId(); + INodeSymlink newNode = dir.addSymlink(id, path, target, modTime, modTime, + new PermissionStatus(userName, null, FsPermission.getDefault())); + if (newNode == null) { + NameNode.stateChangeLog.info("addSymlink: failed to add " + path); + return null; + } + getEditLog().logSymlink(path, target, modTime, modTime, newNode, + logRetryCache); + + if(NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug("addSymlink: " + path + " is added"); + } + return newNode; + } + + /** * Periodically calls hasAvailableResources of NameNodeResourceChecker, and if * there are found to be insufficient resources available, causes the NN to * enter safe mode. If resources are later found to have returned to @@ -4683,6 +4900,21 @@ public class FSNamesystem implements Nam } /** + * Persist the new block (the last block of the given file). + * @param path + * @param file + */ + private void persistNewBlock(String path, INodeFile file) { + Preconditions.checkArgument(file.isUnderConstruction()); + getEditLog().logAddBlock(path, file); + if (NameNode.stateChangeLog.isDebugEnabled()) { + NameNode.stateChangeLog.debug("persistNewBlock: " + + path + " with new block " + file.getLastBlock().toString() + + ", current total block count is " + file.getBlocks().length); + } + } + + /** * SafeModeInfo contains information related to the safe mode. *

* An instance of {@link SafeModeInfo} is created when the name node @@ -6090,7 +6322,7 @@ public class FSNamesystem implements Nam blockinfo.setExpectedLocations(storages); String src = pendingFile.getFullPathName(); - dir.persistBlocks(src, pendingFile, logRetryCache); + persistBlocks(src, pendingFile, logRetryCache); } // rename was successful. If any part of the renamed subtree had @@ -7415,14 +7647,20 @@ public class FSNamesystem implements Nam returnInfo = finalizeRollingUpgradeInternal(now()); getEditLog().logFinalizeRollingUpgrade(returnInfo.getFinalizeTime()); - getFSImage().saveNamespace(this); + if (haEnabled) { + // roll the edit log to make sure the standby NameNode can tail + getFSImage().rollEditLog(); + } getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE); } finally { writeUnlock(); } - // getEditLog().logSync() is not needed since it does saveNamespace + if (!haEnabled) { + // Sync not needed for ha since the edit was rolled after logging. + getEditLog().logSync(); + } if (auditLog.isInfoEnabled() && isExternalInvocation()) { logAuditEvent(true, "finalizeRollingUpgrade", null, null, null); @@ -7718,7 +7956,8 @@ public class FSNamesystem implements Nam checkNameNodeSafeMode("Cannot modify ACL entries on " + src); src = FSDirectory.resolvePath(src, pathComponents, dir); checkOwner(pc, src); - dir.modifyAclEntries(src, aclSpec); + List newAcl = dir.modifyAclEntries(src, aclSpec); + getEditLog().logSetAcl(src, newAcl); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -7739,7 +7978,8 @@ public class FSNamesystem implements Nam checkNameNodeSafeMode("Cannot remove ACL entries on " + src); src = FSDirectory.resolvePath(src, pathComponents, dir); checkOwner(pc, src); - dir.removeAclEntries(src, aclSpec); + List newAcl = dir.removeAclEntries(src, aclSpec); + getEditLog().logSetAcl(src, newAcl); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -7760,7 +8000,8 @@ public class FSNamesystem implements Nam checkNameNodeSafeMode("Cannot remove default ACL entries on " + src); src = FSDirectory.resolvePath(src, pathComponents, dir); checkOwner(pc, src); - dir.removeDefaultAcl(src); + List newAcl = dir.removeDefaultAcl(src); + getEditLog().logSetAcl(src, newAcl); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -7782,6 +8023,7 @@ public class FSNamesystem implements Nam src = FSDirectory.resolvePath(src, pathComponents, dir); checkOwner(pc, src); dir.removeAcl(src); + getEditLog().logSetAcl(src, AclFeature.EMPTY_ENTRY_LIST); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -7802,7 +8044,8 @@ public class FSNamesystem implements Nam checkNameNodeSafeMode("Cannot set ACL on " + src); src = FSDirectory.resolvePath(src, pathComponents, dir); checkOwner(pc, src); - dir.setAcl(src, aclSpec); + List newAcl = dir.setAcl(src, aclSpec); + getEditLog().logSetAcl(src, newAcl); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -7878,7 +8121,8 @@ public class FSNamesystem implements Nam checkOwner(pc, src); checkPathAccess(pc, src, FsAction.WRITE); } - dir.setXAttr(src, xAttr, flag, logRetryCache); + dir.setXAttr(src, xAttr, flag); + getEditLog().logSetXAttr(src, xAttr, logRetryCache); resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); @@ -7999,7 +8243,10 @@ public class FSNamesystem implements Nam checkPathAccess(pc, src, FsAction.WRITE); } - dir.removeXAttr(src, xAttr); + XAttr removedXAttr = dir.removeXAttr(src, xAttr); + if (removedXAttr != null) { + getEditLog().logRemoveXAttr(src, removedXAttr); + } resultingStat = getAuditFileInfo(src, false); } catch (AccessControlException e) { logAuditEvent(false, "removeXAttr", src); Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java Wed Jun 18 05:09:28 2014 @@ -43,6 +43,8 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; +import org.apache.hadoop.io.nativeio.NativeIO; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -132,10 +134,14 @@ public class FileJournalManager implemen Preconditions.checkState(!dstFile.exists(), "Can't finalize edits file " + inprogressFile + " since finalized file " + "already exists"); - if (!inprogressFile.renameTo(dstFile)) { + + try { + NativeIO.renameTo(inprogressFile, dstFile); + } catch (IOException e) { errorReporter.reportErrorOnFile(dstFile); - throw new IllegalStateException("Unable to finalize edits file " + inprogressFile); + throw new IllegalStateException("Unable to finalize edits file " + inprogressFile, e); } + if (inprogressFile.equals(currentInProgress)) { currentInProgress = null; } @@ -513,11 +519,16 @@ public class FileJournalManager implemen File src = file; File dst = new File(src.getParent(), src.getName() + newSuffix); // renameTo fails on Windows if the destination file already exists. - if (!src.renameTo(dst)) { - if (!dst.delete() || !src.renameTo(dst)) { - throw new IOException( - "Couldn't rename log " + src + " to " + dst); + try { + if (dst.exists()) { + if (!dst.delete()) { + throw new IOException("Couldn't delete " + dst); + } } + NativeIO.renameTo(src, dst); + } catch (IOException e) { + throw new IOException( + "Couldn't rename log " + src + " to " + dst, e); } file = dst; } Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Wed Jun 18 05:09:28 2014 @@ -132,6 +132,8 @@ import org.apache.hadoop.ipc.ProtobufRpc import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.WritableRpcEngine; +import org.apache.hadoop.ipc.RefreshRegistry; +import org.apache.hadoop.ipc.RefreshResponse; import org.apache.hadoop.net.Node; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Groups; @@ -147,6 +149,9 @@ import org.apache.hadoop.security.protoc import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB; import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolServerSideTranslatorPB; import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueProtocolService; +import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB; +import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolServerSideTranslatorPB; +import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshProtocolService; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetUserMappingsProtocolService; @@ -229,6 +234,11 @@ class NameNodeRpcServer implements Namen BlockingService refreshCallQueueService = RefreshCallQueueProtocolService .newReflectiveBlockingService(refreshCallQueueXlator); + GenericRefreshProtocolServerSideTranslatorPB genericRefreshXlator = + new GenericRefreshProtocolServerSideTranslatorPB(this); + BlockingService genericRefreshService = GenericRefreshProtocolService + .newReflectiveBlockingService(genericRefreshXlator); + GetUserMappingsProtocolServerSideTranslatorPB getUserMappingXlator = new GetUserMappingsProtocolServerSideTranslatorPB(this); BlockingService getUserMappingService = GetUserMappingsProtocolService @@ -278,6 +288,8 @@ class NameNodeRpcServer implements Namen // We support Refreshing call queue here in case the client RPC queue is full DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class, refreshCallQueueService, serviceRpcServer); + DFSUtil.addPBProtocol(conf, GenericRefreshProtocolPB.class, + genericRefreshService, serviceRpcServer); DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class, getUserMappingService, serviceRpcServer); @@ -322,6 +334,8 @@ class NameNodeRpcServer implements Namen refreshUserMappingService, clientRpcServer); DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class, refreshCallQueueService, clientRpcServer); + DFSUtil.addPBProtocol(conf, GenericRefreshProtocolPB.class, + genericRefreshService, clientRpcServer); DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class, getUserMappingService, clientRpcServer); @@ -1154,6 +1168,12 @@ class NameNodeRpcServer implements Namen serviceRpcServer.refreshCallQueue(conf); } } + + @Override // GenericRefreshProtocol + public Collection refresh(String identifier, String[] args) { + // Let the registry handle as needed + return RefreshRegistry.defaultRegistry().dispatch(identifier, args); + } @Override // GetUserMappingsProtocol public String[] getGroupsForUser(String user) throws IOException { Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocols.java Wed Jun 18 05:09:28 2014 @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.ipc.RefreshCallQueueProtocol; +import org.apache.hadoop.ipc.GenericRefreshProtocol; import org.apache.hadoop.tools.GetUserMappingsProtocol; /** The full set of RPC methods implemented by the Namenode. */ @@ -35,6 +36,7 @@ public interface NamenodeProtocols RefreshAuthorizationPolicyProtocol, RefreshUserMappingsProtocol, RefreshCallQueueProtocol, + GenericRefreshProtocol, GetUserMappingsProtocol, HAServiceProtocol { } Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Wed Jun 18 05:09:28 2014 @@ -40,7 +40,8 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CachePoolStats; -import org.apache.hadoop.hdfs.tools.TableListing.Justification; +import org.apache.hadoop.tools.TableListing; +import org.apache.hadoop.tools.TableListing.Justification; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Wed Jun 18 05:09:28 2014 @@ -26,6 +26,7 @@ import java.net.URL; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -62,12 +63,17 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RefreshCallQueueProtocol; +import org.apache.hadoop.ipc.GenericRefreshProtocol; +import org.apache.hadoop.ipc.RefreshResponse; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; -import org.apache.hadoop.ipc.RefreshCallQueueProtocol; +import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolClientSideTranslatorPB; +import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; @@ -575,7 +581,7 @@ public class DFSAdmin extends FsShell { * @exception IOException * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#restoreFailedStorage(String arg) */ - public int restoreFaileStorage(String arg) throws IOException { + public int restoreFailedStorage(String arg) throws IOException { int exitCode = -1; if(!arg.equals("check") && !arg.equals("true") && !arg.equals("false")) { @@ -688,6 +694,7 @@ public class DFSAdmin extends FsShell { "\t[-refreshUserToGroupsMappings]\n" + "\t[-refreshSuperUserGroupsConfiguration]\n" + "\t[-refreshCallQueue]\n" + + "\t[-refresh [arg1..argn]\n" + "\t[-printTopology]\n" + "\t[-refreshNamenodes datanodehost:port]\n"+ "\t[-deleteBlockPool datanodehost:port blockpoolId [force]]\n"+ @@ -764,6 +771,10 @@ public class DFSAdmin extends FsShell { String refreshCallQueue = "-refreshCallQueue: Reload the call queue from config\n"; + String genericRefresh = "-refresh: Arguments are [arg1..argn]\n" + + "\tTriggers a runtime-refresh of the resource specified by \n" + + "\ton . All other args after are sent to the host."; + String printTopology = "-printTopology: Print a tree of the racks and their\n" + "\t\tnodes as reported by the Namenode\n"; @@ -848,6 +859,8 @@ public class DFSAdmin extends FsShell { System.out.println(refreshSuperUserGroupsConfiguration); } else if ("refreshCallQueue".equals(cmd)) { System.out.println(refreshCallQueue); + } else if ("refresh".equals(cmd)) { + System.out.println(genericRefresh); } else if ("printTopology".equals(cmd)) { System.out.println(printTopology); } else if ("refreshNamenodes".equals(cmd)) { @@ -887,6 +900,7 @@ public class DFSAdmin extends FsShell { System.out.println(refreshUserToGroupsMappings); System.out.println(refreshSuperUserGroupsConfiguration); System.out.println(refreshCallQueue); + System.out.println(genericRefresh); System.out.println(printTopology); System.out.println(refreshNamenodes); System.out.println(deleteBlockPool); @@ -1100,6 +1114,56 @@ public class DFSAdmin extends FsShell { return 0; } + public int genericRefresh(String[] argv, int i) throws IOException { + String hostport = argv[i++]; + String identifier = argv[i++]; + String[] args = Arrays.copyOfRange(argv, i, argv.length); + + // Get the current configuration + Configuration conf = getConf(); + + // for security authorization + // server principal for this call + // should be NN's one. + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, + conf.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "")); + + // Create the client + Class xface = GenericRefreshProtocolPB.class; + InetSocketAddress address = NetUtils.createSocketAddr(hostport); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class); + GenericRefreshProtocolPB proxy = (GenericRefreshProtocolPB) + RPC.getProxy(xface, RPC.getProtocolVersion(xface), address, + ugi, conf, NetUtils.getDefaultSocketFactory(conf), 0); + + GenericRefreshProtocol xlator = + new GenericRefreshProtocolClientSideTranslatorPB(proxy); + + // Refresh + Collection responses = xlator.refresh(identifier, args); + + int returnCode = 0; + + // Print refresh responses + System.out.println("Refresh Responses:\n"); + for (RefreshResponse response : responses) { + System.out.println(response.toString()); + + if (returnCode == 0 && response.getReturnCode() != 0) { + // This is the first non-zero return code, so we should return this + returnCode = response.getReturnCode(); + } else if (returnCode != 0 && response.getReturnCode() != 0) { + // Then now we have multiple non-zero return codes, + // so we merge them into -1 + returnCode = -1; + } + } + + return returnCode; + } + /** * Displays format of commands. * @param cmd The command that is being executed. @@ -1162,6 +1226,9 @@ public class DFSAdmin extends FsShell { } else if ("-refreshCallQueue".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-refreshCallQueue]"); + } else if ("-refresh".equals(cmd)) { + System.err.println("Usage: java DFSAdmin" + + " [-refresh [arg1..argn]"); } else if ("-printTopology".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-printTopology]"); @@ -1195,6 +1262,7 @@ public class DFSAdmin extends FsShell { System.err.println(" [-refreshUserToGroupsMappings]"); System.err.println(" [-refreshSuperUserGroupsConfiguration]"); System.err.println(" [-refreshCallQueue]"); + System.err.println(" [-refresh]"); System.err.println(" [-printTopology]"); System.err.println(" [-refreshNamenodes datanodehost:port]"); System.err.println(" [-deleteBlockPool datanode-host:port blockpoolId [force]]"); @@ -1292,6 +1360,11 @@ public class DFSAdmin extends FsShell { printUsage(cmd); return exitCode; } + } else if ("-refresh".equals(cmd)) { + if (argv.length < 3) { + printUsage(cmd); + return exitCode; + } } else if ("-refreshUserToGroupsMappings".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); @@ -1362,7 +1435,7 @@ public class DFSAdmin extends FsShell { } else if ("-rollEdits".equals(cmd)) { exitCode = rollEdits(); } else if ("-restoreFailedStorage".equals(cmd)) { - exitCode = restoreFaileStorage(argv[i]); + exitCode = restoreFailedStorage(argv[i]); } else if ("-refreshNodes".equals(cmd)) { exitCode = refreshNodes(); } else if ("-finalizeUpgrade".equals(cmd)) { @@ -1387,6 +1460,8 @@ public class DFSAdmin extends FsShell { exitCode = refreshSuperUserGroupsConfiguration(); } else if ("-refreshCallQueue".equals(cmd)) { exitCode = refreshCallQueue(); + } else if ("-refresh".equals(cmd)) { + exitCode = genericRefresh(argv, i); } else if ("-printTopology".equals(cmd)) { exitCode = printTopology(); } else if ("-refreshNamenodes".equals(cmd)) { Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java Wed Jun 18 05:09:28 2014 @@ -34,6 +34,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -54,10 +55,16 @@ import org.apache.hadoop.util.ToolRunner import org.junit.Test; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryType.*; +import static org.apache.hadoop.fs.permission.FsAction.*; +import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.*; +import com.google.common.collect.Lists; + /** * This class tests commands from DFSShell. */ @@ -1620,6 +1627,240 @@ public class TestDFSShell { int res = admin.run(new String[] {"-refreshNodes"}); assertEquals("expected to fail -1", res , -1); } + + // Preserve Copy Option is -ptopxa (timestamps, ownership, permission, XATTR, + // ACLs) + @Test (timeout = 120000) + public void testCopyCommandsWithPreserveOption() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .format(true).build(); + FsShell shell = null; + FileSystem fs = null; + final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithPreserveOption-" + + counter.getAndIncrement(); + final Path hdfsTestDir = new Path(testdir); + try { + fs = cluster.getFileSystem(); + fs.mkdirs(hdfsTestDir); + Path src = new Path(hdfsTestDir, "srcfile"); + fs.create(src).close(); + + fs.setAcl(src, Lists.newArrayList( + aclEntry(ACCESS, USER, ALL), + aclEntry(ACCESS, USER, "foo", ALL), + aclEntry(ACCESS, GROUP, READ_EXECUTE), + aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE), + aclEntry(ACCESS, OTHER, EXECUTE))); + + FileStatus status = fs.getFileStatus(src); + final long mtime = status.getModificationTime(); + final long atime = status.getAccessTime(); + final String owner = status.getOwner(); + final String group = status.getGroup(); + final FsPermission perm = status.getPermission(); + + fs.setXAttr(src, "user.a1", new byte[]{0x31, 0x32, 0x33}); + fs.setXAttr(src, "trusted.a1", new byte[]{0x31, 0x31, 0x31}); + + shell = new FsShell(conf); + + // -p + Path target1 = new Path(hdfsTestDir, "targetfile1"); + String[] argv = new String[] { "-cp", "-p", src.toUri().toString(), + target1.toUri().toString() }; + int ret = ToolRunner.run(shell, argv); + assertEquals("cp -p is not working", SUCCESS, ret); + FileStatus targetStatus = fs.getFileStatus(target1); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + FsPermission targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + Map xattrs = fs.getXAttrs(target1); + assertTrue(xattrs.isEmpty()); + List acls = fs.getAclStatus(target1).getEntries(); + assertTrue(acls.isEmpty()); + assertFalse(targetPerm.getAclBit()); + + // -ptop + Path target2 = new Path(hdfsTestDir, "targetfile2"); + argv = new String[] { "-cp", "-ptop", src.toUri().toString(), + target2.toUri().toString() }; + ret = ToolRunner.run(shell, argv); + assertEquals("cp -ptop is not working", SUCCESS, ret); + targetStatus = fs.getFileStatus(target2); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + xattrs = fs.getXAttrs(target2); + assertTrue(xattrs.isEmpty()); + acls = fs.getAclStatus(target2).getEntries(); + assertTrue(acls.isEmpty()); + assertFalse(targetPerm.getAclBit()); + + // -ptopx + Path target3 = new Path(hdfsTestDir, "targetfile3"); + argv = new String[] { "-cp", "-ptopx", src.toUri().toString(), + target3.toUri().toString() }; + ret = ToolRunner.run(shell, argv); + assertEquals("cp -ptopx is not working", SUCCESS, ret); + targetStatus = fs.getFileStatus(target3); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + xattrs = fs.getXAttrs(target3); + assertEquals(xattrs.size(), 2); + assertArrayEquals(new byte[]{0x31, 0x32, 0x33}, xattrs.get("user.a1")); + assertArrayEquals(new byte[]{0x31, 0x31, 0x31}, xattrs.get("trusted.a1")); + acls = fs.getAclStatus(target3).getEntries(); + assertTrue(acls.isEmpty()); + assertFalse(targetPerm.getAclBit()); + + // -ptopa + Path target4 = new Path(hdfsTestDir, "targetfile4"); + argv = new String[] { "-cp", "-ptopa", src.toUri().toString(), + target4.toUri().toString() }; + ret = ToolRunner.run(shell, argv); + assertEquals("cp -ptopa is not working", SUCCESS, ret); + targetStatus = fs.getFileStatus(target4); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + xattrs = fs.getXAttrs(target4); + assertTrue(xattrs.isEmpty()); + acls = fs.getAclStatus(target4).getEntries(); + assertFalse(acls.isEmpty()); + assertTrue(targetPerm.getAclBit()); + assertEquals(fs.getAclStatus(src), fs.getAclStatus(target4)); + + // -ptoa (verify -pa option will preserve permissions also) + Path target5 = new Path(hdfsTestDir, "targetfile5"); + argv = new String[] { "-cp", "-ptoa", src.toUri().toString(), + target5.toUri().toString() }; + ret = ToolRunner.run(shell, argv); + assertEquals("cp -ptoa is not working", SUCCESS, ret); + targetStatus = fs.getFileStatus(target5); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + xattrs = fs.getXAttrs(target5); + assertTrue(xattrs.isEmpty()); + acls = fs.getAclStatus(target5).getEntries(); + assertFalse(acls.isEmpty()); + assertTrue(targetPerm.getAclBit()); + assertEquals(fs.getAclStatus(src), fs.getAclStatus(target5)); + } finally { + if (null != shell) { + shell.close(); + } + + if (null != fs) { + fs.delete(hdfsTestDir, true); + fs.close(); + } + cluster.shutdown(); + } + } + + // Verify cp -pa option will preserve both ACL and sticky bit. + @Test (timeout = 120000) + public void testCopyCommandsPreserveAclAndStickyBit() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .format(true).build(); + FsShell shell = null; + FileSystem fs = null; + final String testdir = + "/tmp/TestDFSShell-testCopyCommandsPreserveAclAndStickyBit-" + + counter.getAndIncrement(); + final Path hdfsTestDir = new Path(testdir); + try { + fs = cluster.getFileSystem(); + fs.mkdirs(hdfsTestDir); + Path src = new Path(hdfsTestDir, "srcfile"); + fs.create(src).close(); + + fs.setAcl(src, Lists.newArrayList( + aclEntry(ACCESS, USER, ALL), + aclEntry(ACCESS, USER, "foo", ALL), + aclEntry(ACCESS, GROUP, READ_EXECUTE), + aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE), + aclEntry(ACCESS, OTHER, EXECUTE))); + // set sticky bit + fs.setPermission(src, + new FsPermission(ALL, READ_EXECUTE, EXECUTE, true)); + + FileStatus status = fs.getFileStatus(src); + final long mtime = status.getModificationTime(); + final long atime = status.getAccessTime(); + final String owner = status.getOwner(); + final String group = status.getGroup(); + final FsPermission perm = status.getPermission(); + + shell = new FsShell(conf); + + // -p preserves sticky bit and doesn't preserve ACL + Path target1 = new Path(hdfsTestDir, "targetfile1"); + String[] argv = new String[] { "-cp", "-p", src.toUri().toString(), + target1.toUri().toString() }; + int ret = ToolRunner.run(shell, argv); + assertEquals("cp is not working", SUCCESS, ret); + FileStatus targetStatus = fs.getFileStatus(target1); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + FsPermission targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + List acls = fs.getAclStatus(target1).getEntries(); + assertTrue(acls.isEmpty()); + assertFalse(targetPerm.getAclBit()); + + // -ptopa preserves both sticky bit and ACL + Path target2 = new Path(hdfsTestDir, "targetfile2"); + argv = new String[] { "-cp", "-ptopa", src.toUri().toString(), + target2.toUri().toString() }; + ret = ToolRunner.run(shell, argv); + assertEquals("cp -ptopa is not working", SUCCESS, ret); + targetStatus = fs.getFileStatus(target2); + assertEquals(mtime, targetStatus.getModificationTime()); + assertEquals(atime, targetStatus.getAccessTime()); + assertEquals(owner, targetStatus.getOwner()); + assertEquals(group, targetStatus.getGroup()); + targetPerm = targetStatus.getPermission(); + assertTrue(perm.equals(targetPerm)); + acls = fs.getAclStatus(target2).getEntries(); + assertFalse(acls.isEmpty()); + assertTrue(targetPerm.getAclBit()); + assertEquals(fs.getAclStatus(src), fs.getAclStatus(target2)); + } finally { + if (null != shell) { + shell.close(); + } + if (null != fs) { + fs.delete(hdfsTestDir, true); + fs.close(); + } + cluster.shutdown(); + } + } // force Copy Option is -f @Test (timeout = 30000) Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java Wed Jun 18 05:09:28 2014 @@ -25,13 +25,16 @@ import java.io.IOException; import java.io.OutputStream; import java.io.RandomAccessFile; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -453,4 +456,66 @@ public class TestReplication { } fs.delete(fileName, true); } + + /** + * Test that blocks should get replicated if we have corrupted blocks and + * having good replicas at least equal or greater to minreplication + * + * Simulate rbw blocks by creating dummy copies, then a DN restart to detect + * those corrupted blocks asap. + */ + @Test(timeout=30000) + public void testReplicationWhenBlockCorruption() throws Exception { + MiniDFSCluster cluster = null; + try { + Configuration conf = new HdfsConfiguration(); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, 1); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + FileSystem fs = cluster.getFileSystem(); + FSDataOutputStream create = fs.create(new Path("/test")); + fs.setReplication(new Path("/test"), (short) 1); + create.write(new byte[1024]); + create.close(); + + List nonParticipatedNodeDirs = new ArrayList(); + File participatedNodeDirs = null; + for (int i = 0; i < cluster.getDataNodes().size(); i++) { + File storageDir = cluster.getInstanceStorageDir(i, 0); + String bpid = cluster.getNamesystem().getBlockPoolId(); + File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); + if (data_dir.listFiles().length == 0) { + nonParticipatedNodeDirs.add(data_dir); + } else { + participatedNodeDirs = data_dir; + } + } + + String blockFile = null; + File[] listFiles = participatedNodeDirs.listFiles(); + for (File file : listFiles) { + if (file.getName().startsWith("blk_") + && !file.getName().endsWith("meta")) { + blockFile = file.getName(); + for (File file1 : nonParticipatedNodeDirs) { + file1.mkdirs(); + new File(file1, blockFile).createNewFile(); + new File(file1, blockFile + "_1000.meta").createNewFile(); + } + break; + } + } + + fs.setReplication(new Path("/test"), (short) 3); + cluster.restartDataNodes(); // Lets detect all DNs about dummy copied + // blocks + cluster.waitActive(); + cluster.triggerBlockReports(); + DFSTestUtil.waitReplication(fs, new Path("/test"), (short) 3); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java Wed Jun 18 05:09:28 2014 @@ -410,6 +410,7 @@ public abstract class BlockReportTestBas * The second datanode is started in the cluster. * As soon as the replication process is completed test finds a block from * the second DN and sets its GS to be < of original one. + * this is the markBlockAsCorrupt case 3 so we expect one pending deletion * Block report is forced and the check for # of currupted blocks is performed. * Another block is chosen and its length is set to a lesser than original. * A check for another corrupted block is performed after yet another @@ -436,20 +437,20 @@ public abstract class BlockReportTestBas printStats(); assertThat("Wrong number of corrupt blocks", - cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L)); assertThat("Wrong number of PendingDeletion blocks", - cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); assertThat("Wrong number of PendingReplication blocks", cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); - reports = getBlockReports(dn, poolId, true, true); + reports = getBlockReports(dn, poolId, false, true); sendBlockReports(dnR, poolId, reports); printStats(); assertThat("Wrong number of corrupt blocks", - cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L)); + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); assertThat("Wrong number of PendingDeletion blocks", - cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); assertThat("Wrong number of PendingReplication blocks", cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1603355&r1=1603354&r2=1603355&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original) +++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Wed Jun 18 05:09:28 2014 @@ -325,15 +325,14 @@ public class TestBPOfferService { } }).when(mockDn).initBlockPool(Mockito.any(BPOfferService.class)); BPOfferService bpos = setupBPOSForNNs(mockDn, mockNN1, mockNN2); + List actors = bpos.getBPServiceActors(); + assertEquals(2, actors.size()); bpos.start(); try { waitForInitialization(bpos); - List actors = bpos.getBPServiceActors(); - // even if one of the actor initialization fails also other will be - // running until both failed. - assertEquals(2, actors.size()); - BPServiceActor actor = actors.get(0); - waitForBlockReport(actor.getNameNodeProxy()); + // even if one of the actor initialization fails, the other one will be + // finish block report. + waitForBlockReport(mockNN1, mockNN2); } finally { bpos.stop(); } @@ -409,7 +408,32 @@ public class TestBPOfferService { } }, 500, 10000); } - + + private void waitForBlockReport( + final DatanodeProtocolClientSideTranslatorPB mockNN1, + final DatanodeProtocolClientSideTranslatorPB mockNN2) + throws Exception { + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return get(mockNN1) || get(mockNN2); + } + + private Boolean get(DatanodeProtocolClientSideTranslatorPB mockNN) { + try { + Mockito.verify(mockNN).blockReport( + Mockito.anyObject(), + Mockito.eq(FAKE_BPID), + Mockito.anyObject()); + return true; + } catch (Throwable t) { + LOG.info("waiting on block report: " + t.getMessage()); + return false; + } + } + }, 500, 10000); + } + private ReceivedDeletedBlockInfo[] waitForBlockReceived( ExtendedBlock fakeBlock, DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception {