Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F011A200CD0 for ; Tue, 25 Jul 2017 17:53:16 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id EE7D8166F9E; Tue, 25 Jul 2017 15:53:16 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 90D82166F9A for ; Tue, 25 Jul 2017 17:53:15 +0200 (CEST) Received: (qmail 63928 invoked by uid 500); 25 Jul 2017 15:53:14 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 63919 invoked by uid 99); 25 Jul 2017 15:53:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Jul 2017 15:53:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 934E5DFC28; Tue, 25 Jul 2017 15:53:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: brahma@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula. Date: Tue, 25 Jul 2017 15:53:14 +0000 (UTC) archived-at: Tue, 25 Jul 2017 15:53:17 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 1a79dcfc4 -> 6d983cca5 HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d983cca Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d983cca Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d983cca Branch: refs/heads/trunk Commit: 6d983cca52f113118bf49fec527ffb3eb869290a Parents: 1a79dcf Author: Brahma Reddy Battula Authored: Tue Jul 25 23:20:35 2017 +0800 Committer: Brahma Reddy Battula Committed: Tue Jul 25 23:51:53 2017 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/viewfs/Constants.java | 2 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 79 +++++++++++----- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 43 ++++----- .../src/main/resources/core-default.xml | 9 ++ .../conf/TestCommonConfigurationFields.java | 1 + .../hadoop/fs/contract/ContractTestUtils.java | 54 +++++++++++ .../fs/viewfs/ViewFileSystemBaseTest.java | 79 +++++++++++++--- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 94 ++++++++++++++++---- .../fs/viewfs/TestViewFileSystemHdfs.java | 22 +++++ 9 files changed, 309 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index ec8ab2b..9882a8e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -66,4 +66,6 @@ public interface Constants { static public final FsPermission PERMISSION_555 = new FsPermission((short) 0555); + + String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 8265d89..158b099 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -126,7 +126,8 @@ public class ViewFileSystem extends FileSystem { Configuration config; InodeTree fsState; // the fs state; ie the mount table Path homeDir = null; - + // Default to rename within same mountpoint + private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT; /** * Make the path Absolute and get the path-part of a pathname. * Checks that URI matches this file system @@ -207,6 +208,9 @@ public class ViewFileSystem extends FileSystem { } }; workingDir = this.getHomeDirectory(); + renameStrategy = RenameStrategy.valueOf( + conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + RenameStrategy.SAME_MOUNTPOINT.toString())); } catch (URISyntaxException e) { throw new IOException("URISyntax exception: " + theUri); } @@ -490,27 +494,55 @@ public class ViewFileSystem extends FileSystem { if (resDst.isInternalDir()) { throw readOnlyMountTable("rename", dst); } - /** - // Alternate 1: renames within same file system - valid but we disallow - // Alternate 2: (as described in next para - valid but we have disallowed it - // - // Note we compare the URIs. the URIs include the link targets. - // hence we allow renames across mount links as long as the mount links - // point to the same target. - if (!resSrc.targetFileSystem.getUri().equals( - resDst.targetFileSystem.getUri())) { - throw new IOException("Renames across Mount points not supported"); - } - */ - - // - // Alternate 3 : renames ONLY within the the same mount links. - // - if (resSrc.targetFileSystem !=resDst.targetFileSystem) { - throw new IOException("Renames across Mount points not supported"); + + URI srcUri = resSrc.targetFileSystem.getUri(); + URI dstUri = resDst.targetFileSystem.getUri(); + + verifyRenameStrategy(srcUri, dstUri, + resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy); + + ChRootedFileSystem srcFS = (ChRootedFileSystem) resSrc.targetFileSystem; + ChRootedFileSystem dstFS = (ChRootedFileSystem) resDst.targetFileSystem; + return srcFS.getMyFs().rename(srcFS.fullPath(resSrc.remainingPath), + dstFS.fullPath(resDst.remainingPath)); + } + + static void verifyRenameStrategy(URI srcUri, URI dstUri, + boolean isSrcDestSame, ViewFileSystem.RenameStrategy renameStrategy) + throws IOException { + switch (renameStrategy) { + case SAME_FILESYSTEM_ACROSS_MOUNTPOINT: + if (srcUri.getAuthority() != null) { + if (!(srcUri.getScheme().equals(dstUri.getScheme()) && srcUri + .getAuthority().equals(dstUri.getAuthority()))) { + throw new IOException("Renames across Mount points not supported"); + } + } + + break; + case SAME_TARGET_URI_ACROSS_MOUNTPOINT: + // Alternate 2: Rename across mountpoints with same target. + // i.e. Rename across alias mountpoints. + // + // Note we compare the URIs. the URIs include the link targets. + // hence we allow renames across mount links as long as the mount links + // point to the same target. + if (!srcUri.equals(dstUri)) { + throw new IOException("Renames across Mount points not supported"); + } + + break; + case SAME_MOUNTPOINT: + // + // Alternate 3 : renames ONLY within the the same mount links. + // + if (!isSrcDestSame) { + throw new IOException("Renames across Mount points not supported"); + } + break; + default: + throw new IllegalArgumentException ("Unexpected rename strategy"); } - return resSrc.targetFileSystem.rename(resSrc.remainingPath, - resDst.remainingPath); } @Override @@ -1241,4 +1273,9 @@ public class ViewFileSystem extends FileSystem { return allPolicies; } } + + enum RenameStrategy { + SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT, + SAME_FILESYSTEM_ACROSS_MOUNTPOINT + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 3a34a91..364485f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -157,7 +157,9 @@ public class ViewFs extends AbstractFileSystem { final Configuration config; InodeTree fsState; // the fs state; ie the mount table Path homeDir = null; - + private ViewFileSystem.RenameStrategy renameStrategy = + ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT; + static AccessControlException readOnlyMountTable(final String operation, final String p) { return new AccessControlException( @@ -237,6 +239,9 @@ public class ViewFs extends AbstractFileSystem { // return MergeFs.createMergeFs(mergeFsURIList, config); } }; + renameStrategy = ViewFileSystem.RenameStrategy.valueOf( + conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT.toString())); } @Override @@ -495,37 +500,23 @@ public class ViewFs extends AbstractFileSystem { + " is readOnly"); } - InodeTree.ResolveResult resDst = + InodeTree.ResolveResult resDst = fsState.resolve(getUriPath(dst), false); if (resDst.isInternalDir()) { throw new AccessControlException( "Cannot Rename within internal dirs of mount table: dest=" + dst + " is readOnly"); } - - /** - // Alternate 1: renames within same file system - valid but we disallow - // Alternate 2: (as described in next para - valid but we have disallowed it - // - // Note we compare the URIs. the URIs include the link targets. - // hence we allow renames across mount links as long as the mount links - // point to the same target. - if (!resSrc.targetFileSystem.getUri().equals( - resDst.targetFileSystem.getUri())) { - throw new IOException("Renames across Mount points not supported"); - } - */ - - // - // Alternate 3 : renames ONLY within the the same mount links. - // - - if (resSrc.targetFileSystem !=resDst.targetFileSystem) { - throw new IOException("Renames across Mount points not supported"); - } - - resSrc.targetFileSystem.renameInternal(resSrc.remainingPath, - resDst.remainingPath, overwrite); + //Alternate 1: renames within same file system + URI srcUri = resSrc.targetFileSystem.getUri(); + URI dstUri = resDst.targetFileSystem.getUri(); + ViewFileSystem.verifyRenameStrategy(srcUri, dstUri, + resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy); + + ChRootedFs srcFS = (ChRootedFs) resSrc.targetFileSystem; + ChRootedFs dstFS = (ChRootedFs) resDst.targetFileSystem; + srcFS.getMyFs().renameInternal(srcFS.fullPath(resSrc.remainingPath), + dstFS.fullPath(resDst.remainingPath), overwrite); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index d5ddc7f..593fd85 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -801,6 +801,15 @@ + fs.viewfs.rename.strategy + SAME_MOUNTPOINT + Allowed rename strategy to rename between multiple mountpoints. + Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and + SAME_FILESYSTEM_ACROSS_MOUNTPOINT. + + + + fs.AbstractFileSystem.ftp.impl org.apache.hadoop.fs.ftp.FtpFs The FileSystem for Ftp: uris. http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index ef74cba..da37e68 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -95,6 +95,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPropsToSkipCompare.add("nfs3.mountd.port"); xmlPropsToSkipCompare.add("nfs3.server.port"); xmlPropsToSkipCompare.add("test.fs.s3n.name"); + xmlPropsToSkipCompare.add("fs.viewfs.rename.strategy"); // S3N/S3A properties are in a different subtree. // - org.apache.hadoop.fs.s3native.S3NativeFileSystemConfigKeys http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 39c6d18..e60fd43 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.contract; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -718,6 +719,21 @@ public class ContractTestUtils extends Assert { /** * Assert that a file exists and whose {@link FileStatus} entry * declares that this is a file and not a symlink or directory. + * + * @param fileContext filesystem to resolve path against + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + public static void assertIsFile(FileContext fileContext, Path filename) + throws IOException { + assertPathExists(fileContext, "Expected file", filename); + FileStatus status = fileContext.getFileStatus(filename); + assertIsFile(filename, status); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. * @param filename name of the file * @param status file status */ @@ -766,6 +782,25 @@ public class ContractTestUtils extends Assert { } /** + * Assert that a path exists -but make no assertions as to the + * type of that entry. + * + * @param fileContext fileContext to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws FileNotFoundException raised if the path is missing + * @throws IOException IO problems + */ + public static void assertPathExists(FileContext fileContext, String message, + Path path) throws IOException { + if (!fileContext.util().exists(path)) { + //failure, report it + throw new FileNotFoundException( + message + ": not found " + path + " in " + path.getParent()); + } + } + + /** * Assert that a path does not exist. * * @param fileSystem filesystem to examine @@ -786,6 +821,25 @@ public class ContractTestUtils extends Assert { } /** + * Assert that a path does not exist. + * + * @param fileContext fileContext to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist(FileContext fileContext, + String message, Path path) throws IOException { + try { + FileStatus status = fileContext.getFileStatus(path); + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry. * @param fs filesystem * @param dir directory to scan http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 68a7560..db2d2d7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.AclUtil; @@ -51,6 +52,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -366,28 +368,83 @@ abstract public class ViewFileSystemBaseTest { } // rename across mount points that point to same target also fail - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts1() throws IOException { fileSystemTestHelper.createFile(fsView, "/user/foo"); - fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); - /* - code if we had wanted this to succeed - Assert.assertFalse(fSys.exists(new Path("/user/foo"))); - Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo"))); - Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar"))); - Assert.assertTrue(fSysLocal.isFile(new Path(targetTestRoot,"user/fooBarBar"))); - */ + try { + fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } // rename across mount points fail if the mount link targets are different // even if the targets are part of the same target FS - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts2() throws IOException { fileSystemTestHelper.createFile(fsView, "/user/foo"); - fsView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + try { + fsView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } - + + // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled + // to rename across mount points that point to same target URI + @Test + public void testRenameAcrossMounts3() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT + .toString()); + FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2); + fileSystemTestHelper.createFile(fsView2, "/user/foo"); + fsView2.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); + ContractTestUtils + .assertPathDoesNotExist(fsView2, "src should not exist after rename", + new Path("/user/foo")); + ContractTestUtils + .assertPathDoesNotExist(fsTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fsView2, + fileSystemTestHelper.getTestRootPath(fsView2, "/user2/fooBarBar")); + ContractTestUtils + .assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar")); + } + + // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled + // to rename across mount points where the mount link targets are different + // but are part of the same target FS + @Test + public void testRenameAcrossMounts4() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT + .toString()); + FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2); + fileSystemTestHelper.createFile(fsView2, "/user/foo"); + fsView2.rename(new Path("/user/foo"), new Path("/data/fooBar")); + ContractTestUtils + .assertPathDoesNotExist(fsView2, "src should not exist after rename", + new Path("/user/foo")); + ContractTestUtils + .assertPathDoesNotExist(fsTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fsView2, + fileSystemTestHelper.getTestRootPath(fsView2, "/data/fooBar")); + ContractTestUtils + .assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar")); + } + static protected boolean SupportsBlocks = false; // local fs use 1 block // override for HDFS @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index fdc6389..d72ab74 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -58,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.local.LocalConfigKeys; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -66,6 +67,7 @@ import org.apache.hadoop.fs.viewfs.ViewFs.MountPoint; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -345,33 +347,93 @@ abstract public class ViewFsBaseTest { } // rename across mount points that point to same target also fail - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts1() throws IOException { fileContextTestHelper.createFile(fcView, "/user/foo"); - fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); - /* - code if we had wanted this to succeed - Assert.assertFalse(exists(fc, new Path("/user/foo"))); - Assert.assertFalse(exists(fclocal, new Path(targetTestRoot,"user/foo"))); - Assert.assertTrue(isFile(fc, - FileContextTestHelper.getTestRootPath(fc,"/user2/fooBarBar"))); - Assert.assertTrue(isFile(fclocal, - new Path(targetTestRoot,"user/fooBarBar"))); - */ + try { + fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } // rename across mount points fail if the mount link targets are different // even if the targets are part of the same target FS - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts2() throws IOException { fileContextTestHelper.createFile(fcView, "/user/foo"); - fcView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + try { + fcView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } - - - - + + // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled + // to rename across mount points that point to same target URI + @Test + public void testRenameAcrossMounts3() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT + .toString()); + + FileContext fcView2 = + FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2); + String user1Path = "/user/foo"; + fileContextTestHelper.createFile(fcView2, user1Path); + String user2Path = "/user2/fooBarBar"; + Path user2Dst = new Path(user2Path); + fcView2.rename(new Path(user1Path), user2Dst); + ContractTestUtils + .assertPathDoesNotExist(fcView2, "src should not exist after rename", + new Path(user1Path)); + ContractTestUtils + .assertPathDoesNotExist(fcTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fcView2, + fileContextTestHelper.getTestRootPath(fcView2, user2Path)); + ContractTestUtils + .assertIsFile(fcTarget, new Path(targetTestRoot, "user/fooBarBar")); + } + + // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled + // to rename across mount points if the mount link targets are different + // but are part of the same target FS + @Test + public void testRenameAcrossMounts4() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT + .toString()); + FileContext fcView2 = + FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2); + String userPath = "/user/foo"; + fileContextTestHelper.createFile(fcView2, userPath); + String anotherMountPath = "/data/fooBar"; + Path anotherDst = new Path(anotherMountPath); + fcView2.rename(new Path(userPath), anotherDst); + + ContractTestUtils + .assertPathDoesNotExist(fcView2, "src should not exist after rename", + new Path(userPath)); + ContractTestUtils + .assertPathDoesNotExist(fcTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fcView2, + fileContextTestHelper.getTestRootPath(fcView2, anotherMountPath)); + ContractTestUtils + .assertIsFile(fcView2, new Path(targetTestRoot, "data/fooBar")); + } + static protected boolean SupportsBlocks = false; // local fs use 1 block // override for HDFS @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d983cca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 58b77f6..b8f5379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -247,4 +249,24 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { Assert.assertTrue("File checksum not matching!", fileChecksumViaViewFs.equals(fileChecksumViaTargetFs)); } + + //Rename should fail on across different fileSystems + @Test + public void testRenameAccorssFilesystem() throws IOException { + //data is mountpoint in nn1 + Path mountDataRootPath = new Path("/data"); + //mountOnNn2 is nn2 mountpoint + Path fsTargetFilePath = new Path("/mountOnNn2"); + Path filePath = new Path(mountDataRootPath + "/ttest"); + Path hdfFilepath = new Path(fsTargetFilePath + "/ttest2"); + fsView.create(filePath); + try { + fsView.rename(filePath, hdfFilepath); + ContractTestUtils.fail("Should thrown IOE on Renames across filesytems"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org