Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4AA68200BB4 for ; Tue, 18 Oct 2016 01:14:28 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4926F160AF0; Mon, 17 Oct 2016 23:14:28 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6A23A160AFB for ; Tue, 18 Oct 2016 01:14:27 +0200 (CEST) Received: (qmail 3117 invoked by uid 500); 17 Oct 2016 23:14:21 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 2492 invoked by uid 99); 17 Oct 2016 23:14:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Oct 2016 23:14:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 603BBDFBA8; Mon, 17 Oct 2016 23:14:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: umamahesh@apache.org To: common-commits@hadoop.apache.org Date: Mon, 17 Oct 2016 23:14:42 -0000 Message-Id: <326e93f551af46538db557f1f9ac82de@git.apache.org> In-Reply-To: <91cf63af350e4d6186f9f8359f6fdc63@git.apache.org> References: <91cf63af350e4d6186f9f8359f6fdc63@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [23/50] hadoop git commit: HADOOP-13024. Distcp with -delete feature on raw data not implemented. Contributed by Mavin Martin. archived-at: Mon, 17 Oct 2016 23:14:28 -0000 HADOOP-13024. Distcp with -delete feature on raw data not implemented. Contributed by Mavin Martin. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a85d079 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a85d079 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a85d079 Branch: refs/heads/HDFS-10285 Commit: 0a85d079838f532a13ca237300386d1b3bc1b178 Parents: 8c721aa Author: Jing Zhao Authored: Thu Oct 13 13:24:37 2016 -0700 Committer: Jing Zhao Committed: Thu Oct 13 13:24:54 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/tools/DistCpConstants.java | 12 +++++- .../hadoop/tools/mapred/CopyCommitter.java | 5 ++- .../hadoop/tools/TestDistCpWithRawXAttrs.java | 45 +++++++++----------- .../hadoop/tools/util/DistCpTestUtils.java | 32 ++++++++------ 4 files changed, 56 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java index 96f364c..6171aa9 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java @@ -18,6 +18,8 @@ package org.apache.hadoop.tools; * limitations under the License. */ +import org.apache.hadoop.fs.Path; + /** * Utility class to hold commonly used constants. */ @@ -125,9 +127,17 @@ public class DistCpConstants { public static final int SPLIT_RATIO_DEFAULT = 2; /** + * Constants for NONE file deletion + */ + public static final String NONE_PATH_NAME = "/NONE"; + public static final Path NONE_PATH = new Path(NONE_PATH_NAME); + public static final Path RAW_NONE_PATH = new Path( + DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME + NONE_PATH_NAME); + + /** * Value of reserved raw HDFS directory when copying raw.* xattrs. */ - static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw"; + public static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw"; static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 6d2fef5..dd653b2 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -238,7 +238,10 @@ public class CopyCommitter extends FileOutputCommitter { List targets = new ArrayList(1); Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); targets.add(targetFinalPath); - DistCpOptions options = new DistCpOptions(targets, new Path("/NONE")); + Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath) + .toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME) + ? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH; + DistCpOptions options = new DistCpOptions(targets, resultNonePath); // // Set up options to be the same from the CopyListing.buildListing's perspective, // so to collect similar listings as when doing the copy http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java index 5aef51a..8adc2cf 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java @@ -82,14 +82,7 @@ public class TestDistCpWithRawXAttrs { final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest"; doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-px", - false, true, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-px", - false, true, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-px", - false, true, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-px", - true, true, DistCpConstants.SUCCESS); + doTestStandardPreserveRawXAttrs("-px", true); final Path savedWd = fs.getWorkingDirectory(); try { fs.setWorkingDirectory(new Path("/.reserved/raw")); @@ -103,27 +96,18 @@ public class TestDistCpWithRawXAttrs { /* Test that XAttrs are not preserved and raw.* are when appropriate. */ @Test public void testPreserveRawXAttrs2() throws Exception { - doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-p", - false, false, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-p", - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-p", - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-p", - true, false, DistCpConstants.SUCCESS); + doTestStandardPreserveRawXAttrs("-p", false); } /* Test that XAttrs are not preserved and raw.* are when appropriate. */ @Test public void testPreserveRawXAttrs3() throws Exception { - doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, null, - false, false, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rawDestName, null, - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rootedDestName, null, - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rawDestName, null, - true, false, DistCpConstants.SUCCESS); + doTestStandardPreserveRawXAttrs(null, false); + } + + @Test + public void testPreserveRawXAttrs4() throws Exception { + doTestStandardPreserveRawXAttrs("-update -delete", false); } private static Path[] pathnames = { new Path("dir1"), @@ -145,6 +129,19 @@ public class TestDistCpWithRawXAttrs { } } + private void doTestStandardPreserveRawXAttrs(String options, + boolean expectUser) + throws Exception { + doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, options, + false, expectUser, DistCpConstants.SUCCESS); + doTestPreserveRawXAttrs(rootedSrcName, rawDestName, options, + false, expectUser, DistCpConstants.INVALID_ARGUMENT); + doTestPreserveRawXAttrs(rawSrcName, rootedDestName, options, + false, expectUser, DistCpConstants.INVALID_ARGUMENT); + doTestPreserveRawXAttrs(rawSrcName, rawDestName, options, + true, expectUser, DistCpConstants.SUCCESS); + } + private void doTestPreserveRawXAttrs(String src, String dest, String preserveOpts, boolean expectRaw, boolean expectUser, int expectedExitCode) throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java index 2721638..624f7d5 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java @@ -18,21 +18,20 @@ package org.apache.hadoop.tools.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; - import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.util.ToolRunner; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Utility class for DistCpTests */ @@ -79,10 +78,19 @@ public class DistCpTestUtils { public static void assertRunDistCp(int exitCode, String src, String dst, String options, Configuration conf) throws Exception { + assertRunDistCp(exitCode, src, dst, + options == null ? new String[0] : options.trim().split(" "), conf); + } + + private static void assertRunDistCp(int exitCode, String src, String dst, + String[] options, Configuration conf) + throws Exception { DistCp distCp = new DistCp(conf, null); - String[] optsArr = options == null ? - new String[] { src, dst } : - new String[] { options, src, dst }; + String[] optsArr = new String[options.length + 2]; + System.arraycopy(options, 0, optsArr, 0, options.length); + optsArr[optsArr.length - 2] = src; + optsArr[optsArr.length - 1] = dst; + assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr)); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org