Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A9FCF200B4F for ; Tue, 12 Jul 2016 03:07:25 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A8936160A7D; Tue, 12 Jul 2016 01:07:25 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C8F50160A78 for ; Tue, 12 Jul 2016 03:07:24 +0200 (CEST) Received: (qmail 54939 invoked by uid 500); 12 Jul 2016 01:07:23 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54930 invoked by uid 99); 12 Jul 2016 01:07:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 12 Jul 2016 01:07:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C9426DFFF8; Tue, 12 Jul 2016 01:07:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wang@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-10300. TestDistCpSystem should share MiniDFSCluster. Contributed by John Zhuge. Date: Tue, 12 Jul 2016 01:07:23 +0000 (UTC) archived-at: Tue, 12 Jul 2016 01:07:25 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 1b7691b4c -> 4e7e48cdc HDFS-10300. TestDistCpSystem should share MiniDFSCluster. Contributed by John Zhuge. (cherry picked from commit f292624bd8dbdc1841f225a34346d0392fa76a47) (cherry picked from commit 38b1eafdbcbd91bd806b842729b84d84f3600524) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e7e48cd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e7e48cd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e7e48cd Branch: refs/heads/branch-2.8 Commit: 4e7e48cdc1a5d3e6e9a4c5988c3db84b9be9e000 Parents: 1b7691b Author: Andrew Wang Authored: Mon Jul 11 18:06:24 2016 -0700 Committer: Andrew Wang Committed: Mon Jul 11 18:06:45 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/tools/TestDistCpSystem.java | 199 ++++++++++--------- 1 file changed, 107 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e7e48cd/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java index cd86560..e3018a0 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java @@ -18,7 +18,10 @@ package org.apache.hadoop.tools; +import static org.apache.hadoop.test.GenericTestUtils.getMethodName; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.io.OutputStream; @@ -26,26 +29,33 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.util.ToolRunner; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; /** * A JUnit test for copying files recursively. */ -public class TestDistCpSystem extends TestCase { - +public class TestDistCpSystem { + @Rule + public Timeout globalTimeout = new Timeout(30000); + private static final String SRCDAT = "srcdat"; private static final String DSTDAT = "dstdat"; - + + private static MiniDFSCluster cluster; + private static Configuration conf; + private class FileEntry { String path; boolean isDir; @@ -92,60 +102,66 @@ public class TestDistCpSystem extends TestCase { private static void deldir(FileSystem fs, String topdir) throws IOException { fs.delete(new Path(topdir), true); } - - private void testPreserveUserHelper( - FileEntry[] srcEntries, - FileEntry[] dstEntries, - boolean createSrcDir, - boolean createTgtDir, - boolean update) throws Exception { - Configuration conf = null; - MiniDFSCluster cluster = null; - try { - final String testRoot = "/testdir"; - final String testSrcRel = SRCDAT; - final String testSrc = testRoot + "/" + testSrcRel; - final String testDstRel = DSTDAT; - final String testDst = testRoot + "/" + testDstRel; - - conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - - String nnUri = FileSystem.getDefaultUri(conf).toString(); - FileSystem fs = FileSystem.get(URI.create(nnUri), conf); - fs.mkdirs(new Path(testRoot)); - if (createSrcDir) { - fs.mkdirs(new Path(testSrc)); - } - if (createTgtDir) { - fs.mkdirs(new Path(testDst)); - } - - createFiles(fs, testRoot, srcEntries); - FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries); - for(int i = 0; i < srcEntries.length; i++) { - fs.setOwner(srcstats[i].getPath(), "u" + i, null); - } - String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, - nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; - - ToolRunner.run(conf, new DistCp(), args); - - String realTgtPath = testDst; - if (!createTgtDir) { - realTgtPath = testRoot; - } - FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries); - for(int i = 0; i < dststat.length; i++) { - assertEquals("i=" + i, "u" + i, dststat[i].getOwner()); - } - deldir(fs, testRoot); - } finally { - if (cluster != null) { cluster.shutdown(); } + + private void testPreserveUserHelper(String testRoot, + FileEntry[] srcEntries, + FileEntry[] dstEntries, + boolean createSrcDir, + boolean createTgtDir, + boolean update) throws Exception { + final String testSrcRel = SRCDAT; + final String testSrc = testRoot + "/" + testSrcRel; + final String testDstRel = DSTDAT; + final String testDst = testRoot + "/" + testDstRel; + + String nnUri = FileSystem.getDefaultUri(conf).toString(); + FileSystem fs = FileSystem.get(URI.create(nnUri), conf); + fs.mkdirs(new Path(testRoot)); + if (createSrcDir) { + fs.mkdirs(new Path(testSrc)); + } + if (createTgtDir) { + fs.mkdirs(new Path(testDst)); + } + + createFiles(fs, testRoot, srcEntries); + FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries); + for(int i = 0; i < srcEntries.length; i++) { + fs.setOwner(srcstats[i].getPath(), "u" + i, null); } + String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, + nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; + + ToolRunner.run(conf, new DistCp(), args); + + String realTgtPath = testDst; + if (!createTgtDir) { + realTgtPath = testRoot; + } + FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries); + for(int i = 0; i < dststat.length; i++) { + assertEquals("i=" + i, "u" + i, dststat[i].getOwner()); + } + deldir(fs, testRoot); } + @BeforeClass + public static void beforeClass() throws IOException { + conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + } + + @AfterClass + public static void afterClass() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test public void testPreserveUseNonEmptyDir() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT, true), new FileEntry(SRCDAT + "/a", false), @@ -160,12 +176,14 @@ public class TestDistCpSystem extends TestCase { new FileEntry(DSTDAT + "/b/c", false) }; - testPreserveUserHelper(srcfiles, srcfiles, false, true, false); - testPreserveUserHelper(srcfiles, dstfiles, false, false, false); + testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } - - + + + @Test public void testPreserveUserEmptyDir() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT, true) }; @@ -174,22 +192,26 @@ public class TestDistCpSystem extends TestCase { new FileEntry(DSTDAT, true) }; - testPreserveUserHelper(srcfiles, srcfiles, false, true, false); - testPreserveUserHelper(srcfiles, dstfiles, false, false, false); + testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } + @Test public void testPreserveUserSingleFile() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT, false) }; FileEntry[] dstfiles = { new FileEntry(DSTDAT, false) }; - testPreserveUserHelper(srcfiles, srcfiles, false, true, false); - testPreserveUserHelper(srcfiles, dstfiles, false, false, false); + testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } - + + @Test public void testPreserveUserNonEmptyDirWithUpdate() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT + "/a", false), new FileEntry(SRCDAT + "/b", true), @@ -202,37 +224,30 @@ public class TestDistCpSystem extends TestCase { new FileEntry("b/c", false) }; - testPreserveUserHelper(srcfiles, dstfiles, true, true, true); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, true, true, true); } @Test public void testSourceRoot() throws Exception { - MiniDFSCluster cluster = null; - Configuration conf = new Configuration(); - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); - FileSystem fs = cluster.getFileSystem(); - - String rootStr = fs.makeQualified(new Path("/")).toString(); - - // Case 1. The target does not exist. - - String tgtStr = fs.makeQualified(new Path("/nodir")).toString(); - String[] args = new String[]{ rootStr, tgtStr }; - Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0)); - - // Case 2. The target exists. - - Path tgtPath2 = new Path("/dir"); - assertTrue(fs.mkdirs(tgtPath2)); - String tgtStr2 = fs.makeQualified(tgtPath2).toString(); - String[] args2 = new String[]{ rootStr, tgtStr2 }; - Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0)); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } + FileSystem fs = cluster.getFileSystem(); + + String rootStr = fs.makeQualified(new Path("/")).toString(); + + String testRoot = "/testdir." + getMethodName(); + + // Case 1. The target does not exist. + + Path tgtPath = new Path(testRoot + "/nodir"); + String tgtStr = fs.makeQualified(tgtPath).toString(); + String[] args = new String[]{rootStr, tgtStr}; + Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0)); + + // Case 2. The target exists. + + Path tgtPath2 = new Path(testRoot + "/dir"); + assertTrue(fs.mkdirs(tgtPath2)); + String tgtStr2 = fs.makeQualified(tgtPath2).toString(); + String[] args2 = new String[]{rootStr, tgtStr2}; + Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0)); } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org