From common-commits-return-79418-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Wed Mar 7 04:42:09 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id A0FF3180652 for ; Wed, 7 Mar 2018 04:42:08 +0100 (CET) Received: (qmail 23943 invoked by uid 500); 7 Mar 2018 03:42:06 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 23931 invoked by uid 99); 7 Mar 2018 03:42:06 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Mar 2018 03:42:06 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E3B4BF6485; Wed, 7 Mar 2018 03:42:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xyao@apache.org To: common-commits@hadoop.apache.org Message-Id: <584be5b0f31e461eba4c5a44de597f80@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-13109. Support fully qualified hdfs path in EZ commands. Contributed by Hanisha Koneru. Date: Wed, 7 Mar 2018 03:42:05 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.9 ef54f7853 -> 321766bcc HDFS-13109. Support fully qualified hdfs path in EZ commands. Contributed by Hanisha Koneru. (cherry picked from commit edf9445708ffb7a9e59cb933e049b540f99add1e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/321766bc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/321766bc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/321766bc Branch: refs/heads/branch-2.9 Commit: 321766bcc8f8c7ac3d0091a065ccabf8a16c13c5 Parents: ef54f78 Author: Xiaoyu Yao Authored: Tue Mar 6 16:44:20 2018 -0800 Committer: Xiaoyu Yao Committed: Tue Mar 6 18:33:11 2018 -0800 ---------------------------------------------------------------------- .../hadoop/hdfs/DistributedFileSystem.java | 65 ++++++++++++++++++++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 46 +------------- .../apache/hadoop/hdfs/TestEncryptionZones.java | 44 +++++++++++++ 3 files changed, 111 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/321766bc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 497f770..46df7d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStreamBuilder; import org.apache.hadoop.fs.FSLinkResolver; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; @@ -2395,6 +2396,70 @@ public class DistributedFileSystem extends FileSystem }.resolve(this, absF); } + /* HDFS only */ + public void provisionEZTrash(final Path path, + final FsPermission trashPermission) throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(Path p) throws IOException { + provisionEZTrash(getPathName(p), trashPermission); + return null; + } + + @Override + public Void next(FileSystem fs, Path p) throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + myDfs.provisionEZTrash(p, trashPermission); + return null; + } + throw new UnsupportedOperationException("Cannot provisionEZTrash " + + "through a symlink to a non-DistributedFileSystem: " + fs + " -> " + + p); + } + }.resolve(this, absF); + } + + private void provisionEZTrash(String path, FsPermission trashPermission) + throws IOException { + // make sure the path is an EZ + EncryptionZone ez = dfs.getEZForPath(path); + if (ez == null) { + throw new IllegalArgumentException(path + " is not an encryption zone."); + } + + String ezPath = ez.getPath(); + if (!path.toString().equals(ezPath)) { + throw new IllegalArgumentException(path + " is not the root of an " + + "encryption zone. Do you mean " + ez.getPath() + "?"); + } + + // check if the trash directory exists + Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX); + try { + FileStatus trashFileStatus = getFileStatus(trashPath); + String errMessage = "Will not provision new trash directory for " + + "encryption zone " + ez.getPath() + ". Path already exists."; + if (!trashFileStatus.isDirectory()) { + errMessage += "\r\n" + + "Warning: " + trashPath.toString() + " is not a directory"; + } + if (!trashFileStatus.getPermission().equals(trashPermission)) { + errMessage += "\r\n" + + "Warning: the permission of " + + trashPath.toString() + " is not " + trashPermission; + } + throw new FileAlreadyExistsException(errMessage); + } catch (FileNotFoundException ignored) { + // no trash path + } + + // Update the permission bits + mkdir(trashPath, trashPermission); + setPermission(trashPath, trashPermission); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet flag) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/321766bc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 4c37889..2e40358 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -30,9 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CacheFlag; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -324,7 +322,7 @@ public class HdfsAdmin { throw new HadoopIllegalArgumentException( "can not have both PROVISION_TRASH and NO_TRASH flags"); } - this.provisionEZTrash(path); + dfs.provisionEZTrash(path, TRASH_PERMISSION); } } @@ -335,7 +333,7 @@ public class HdfsAdmin { * @throws IOException if the trash directory can not be created. */ public void provisionEncryptionZoneTrash(Path path) throws IOException { - this.provisionEZTrash(path); + dfs.provisionEZTrash(path, TRASH_PERMISSION); } /** @@ -473,46 +471,6 @@ public class HdfsAdmin { return dfs.getAllStoragePolicies(); } - private void provisionEZTrash(Path path) throws IOException { - // make sure the path is an EZ - EncryptionZone ez = dfs.getEZForPath(path); - if (ez == null) { - throw new IllegalArgumentException(path + " is not an encryption zone."); - } - - String ezPath = ez.getPath(); - if (!path.toString().equals(ezPath)) { - throw new IllegalArgumentException(path + " is not the root of an " + - "encryption zone. Do you mean " + ez.getPath() + "?"); - } - - // check if the trash directory exists - - Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX); - - try { - FileStatus trashFileStatus = dfs.getFileStatus(trashPath); - String errMessage = "Will not provision new trash directory for " + - "encryption zone " + ez.getPath() + ". Path already exists."; - if (!trashFileStatus.isDirectory()) { - errMessage += "\r\n" + - "Warning: " + trashPath.toString() + " is not a directory"; - } - if (!trashFileStatus.getPermission().equals(TRASH_PERMISSION)) { - errMessage += "\r\n" + - "Warning: the permission of " + - trashPath.toString() + " is not " + TRASH_PERMISSION; - } - throw new FileAlreadyExistsException(errMessage); - } catch (FileNotFoundException ignored) { - // no trash path - } - - // Update the permission bits - dfs.mkdir(trashPath, TRASH_PERMISSION); - dfs.setPermission(trashPath, TRASH_PERMISSION); - } - /** * Returns a RemoteIterator which can be used to list all open files * currently managed by the NameNode. For large numbers of open files, http://git-wip-us.apache.org/repos/asf/hadoop/blob/321766bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index eac6d23..d3dc110 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -538,6 +538,50 @@ public class TestEncryptionZones { assertZonePresent(null, rootDir.toString()); } + @Test + public void testEZwithFullyQualifiedPath() throws Exception { + /* Test failure of create EZ on a directory that doesn't exist. */ + final Path zoneParent = new Path("/zones"); + final Path zone1 = new Path(zoneParent, "zone1"); + final Path zone1FQP = new Path(cluster.getURI().toString(), zone1); + final Path zone2 = new Path(zoneParent, "zone2"); + final Path zone2FQP = new Path(cluster.getURI().toString(), zone2); + + int numZones = 0; + EnumSet withTrash = EnumSet + .of(CreateEncryptionZoneFlag.PROVISION_TRASH); + + // Create EZ with Trash using FQP + fsWrapper.mkdir(zone1FQP, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone1FQP, TEST_KEY, withTrash); + assertNumZones(++numZones); + assertZonePresent(TEST_KEY, zone1.toString()); + // Check that zone1 contains a .Trash directory + final Path zone1Trash = new Path(zone1, fs.TRASH_PREFIX); + assertTrue("CreateEncryptionZone with trash enabled should create a " + + ".Trash directory in the EZ", fs.exists(zone1Trash)); + + // getEncryptionZoneForPath for FQP should return the path component + EncryptionZone ezForZone1 = dfsAdmin.getEncryptionZoneForPath(zone1FQP); + assertTrue("getEncryptionZoneForPath for fully qualified path should " + + "return the path component", + ezForZone1.getPath().equals(zone1.toString())); + + // Create EZ without Trash + fsWrapper.mkdir(zone2FQP, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone2FQP, TEST_KEY, NO_TRASH); + assertNumZones(++numZones); + assertZonePresent(TEST_KEY, zone2.toString()); + + // Provision Trash on zone2 using FQP + dfsAdmin.provisionEncryptionZoneTrash(zone2FQP); + EncryptionZone ezForZone2 = dfsAdmin.getEncryptionZoneForPath(zone2FQP); + Path ezTrashForZone2 = new Path(ezForZone2.getPath(), + FileSystem.TRASH_PREFIX); + assertTrue("provisionEZTrash with fully qualified path should create " + + "trash directory ", fsWrapper.exists(ezTrashForZone2)); + } + /** * Test listing encryption zones as a non super user. */ --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org