Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 506F319E4E for ; Thu, 28 Apr 2016 18:01:57 +0000 (UTC) Received: (qmail 85873 invoked by uid 500); 28 Apr 2016 18:01:46 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 85420 invoked by uid 500); 28 Apr 2016 18:01:46 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 84802 invoked by uid 99); 28 Apr 2016 18:01:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Apr 2016 18:01:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0D186E0381; Thu, 28 Apr 2016 18:01:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Date: Thu, 28 Apr 2016 18:02:00 -0000 Message-Id: In-Reply-To: <58f146a61c66461384583b583997bb9c@git.apache.org> References: <58f146a61c66461384583b583997bb9c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [16/50] [abbrv] hadoop git commit: HDFS-10313. Distcp need to enforce the order of snapshot names passed to -diff. (Lin Yiqun via Yongjun Zhang) HDFS-10313. Distcp need to enforce the order of snapshot names passed to -diff. (Lin Yiqun via Yongjun Zhang) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/959a28dd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/959a28dd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/959a28dd Branch: refs/heads/YARN-3368 Commit: 959a28dd1216dfac78d05b438828e8503108d963 Parents: 6be22dd Author: Yongjun Zhang Authored: Tue Apr 26 16:08:03 2016 -0700 Committer: Yongjun Zhang Committed: Tue Apr 26 16:08:03 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/tools/CopyListing.java | 4 +++ .../java/org/apache/hadoop/tools/DistCp.java | 3 ++- .../org/apache/hadoop/tools/DistCpSync.java | 26 +++++++++++++++++++ .../org/apache/hadoop/tools/TestDistCpSync.java | 27 ++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/959a28dd/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java index 1efc56c..481aa61 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java @@ -268,6 +268,10 @@ public abstract class CopyListing extends Configured { public InvalidInputException(String message) { super(message); } + + public InvalidInputException(String message, Throwable cause) { + super(message, cause); + } } public static class AclsNotSupportedException extends RuntimeException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/959a28dd/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 2342fa9..36e5ee4 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -179,7 +179,8 @@ public class DistCp extends Configured implements Tool { if (distCpSync.sync()) { createInputFileListingWithDiff(job, distCpSync); } else { - inputOptions.disableUsingDiff(); + throw new Exception("DistCp sync failed, input options: " + + inputOptions); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/959a28dd/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index 2b8ed71..38a1bef 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -17,14 +17,18 @@ */ package org.apache.hadoop.tools; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.tools.CopyListing.InvalidInputException; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -91,6 +95,28 @@ class DistCpSync { inputOptions.getToSnapshot()))); return false; } + + final String from = getSnapshotName(inputOptions.getFromSnapshot()); + final String to = getSnapshotName(inputOptions.getToSnapshot()); + + try { + final FileStatus fromSnapshotStat = + sfs.getFileStatus(getSourceSnapshotPath(sourceDir, from)); + + final FileStatus toSnapshotStat = + sfs.getFileStatus(getSourceSnapshotPath(sourceDir, to)); + + // If toSnapshot isn't current dir then do a time check + if (!to.equals("") + && fromSnapshotStat.getModificationTime() > toSnapshotStat + .getModificationTime()) { + throw new HadoopIllegalArgumentException("Snapshot " + to + + " should be newer than " + from); + } + } catch (FileNotFoundException nfe) { + throw new InvalidInputException("Input snapshot is not found", nfe); + } + return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/959a28dd/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 90e6840..3419b2f 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.tools; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -31,6 +32,7 @@ import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.tools.mapred.CopyMapper; import org.junit.After; import org.junit.Assert; @@ -712,4 +714,29 @@ public class TestDistCpSync { verifyCopy(dfs.getFileStatus(sourcePath), dfs.getFileStatus(target), false); } + + @Test + public void testSyncSnapshotTimeStampChecking() throws Exception { + initData(source); + initData(target); + dfs.allowSnapshot(source); + dfs.allowSnapshot(target); + dfs.createSnapshot(source, "s2"); + dfs.createSnapshot(target, "s1"); + // Sleep one second to make snapshot s1 created later than s2 + Thread.sleep(1000); + dfs.createSnapshot(source, "s1"); + + boolean threwException = false; + try { + DistCpSync distCpSync = new DistCpSync(options, conf); + // do the sync + distCpSync.sync(); + } catch (HadoopIllegalArgumentException e) { + threwException = true; + GenericTestUtils.assertExceptionContains( + "Snapshot s2 should be newer than s1", e); + } + Assert.assertTrue(threwException); + } }