Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4B416183C5 for ; Fri, 9 Oct 2015 18:49:36 +0000 (UTC) Received: (qmail 77116 invoked by uid 500); 9 Oct 2015 18:49:33 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 77080 invoked by uid 500); 9 Oct 2015 18:49:32 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77071 invoked by uid 99); 9 Oct 2015 18:49:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 Oct 2015 18:49:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 68A52DFDD7; Fri, 9 Oct 2015 18:49:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wang@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-9110. Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for better efficiency. Contributed by Charlie Helin. Date: Fri, 9 Oct 2015 18:49:32 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk de8efc65a -> 357b1fd08 HDFS-9110. Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for better efficiency. Contributed by Charlie Helin. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/357b1fd0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/357b1fd0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/357b1fd0 Branch: refs/heads/trunk Commit: 357b1fd0822447f9e73a20c69f37006d9a37ecbc Parents: de8efc6 Author: Andrew Wang Authored: Fri Oct 9 11:49:16 2015 -0700 Committer: Andrew Wang Committed: Fri Oct 9 11:49:16 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/namenode/NNUpgradeUtil.java | 48 ++++++++++++-------- 2 files changed, 33 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/357b1fd0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ce8d79a..24c359b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1503,6 +1503,9 @@ Release 2.8.0 - UNRELEASED HDFS-9181. Better handling of exceptions thrown during upgrade shutdown. (Wei-Chiu Chuang via Yongjun Zhang) + HDFS-9110. Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for + better efficiency. (Charlie Helin via wang) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/357b1fd0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java index 1f10bc4..b3fff74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java @@ -18,10 +18,14 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.File; -import java.io.FilenameFilter; import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; import java.nio.file.Files; -import java.util.List; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,7 +35,6 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; import com.google.common.base.Preconditions; -import org.apache.hadoop.io.IOUtils; public abstract class NNUpgradeUtil { @@ -116,21 +119,30 @@ public abstract class NNUpgradeUtil { // rename current to tmp renameCurToTmp(sd); - final File curDir = sd.getCurrentDir(); - final File tmpDir = sd.getPreviousTmp(); - List fileNameList = IOUtils.listDirectory(tmpDir, new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return dir.equals(tmpDir) - && name.startsWith(NNStorage.NameNodeFile.EDITS.getName()); - } - }); - - for (String s : fileNameList) { - File prevFile = new File(tmpDir, s); - File newFile = new File(curDir, prevFile.getName()); - Files.createLink(newFile.toPath(), prevFile.toPath()); - } + final Path curDir = sd.getCurrentDir().toPath(); + final Path tmpDir = sd.getPreviousTmp().toPath(); + + Files.walkFileTree(tmpDir, + /* do not follow links */ Collections.emptySet(), + 1, new SimpleFileVisitor() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + + String name = file.getFileName().toString(); + + if (Files.isRegularFile(file) + && name.startsWith(NNStorage.NameNodeFile.EDITS.getName())) { + + Path newFile = curDir.resolve(name); + Files.createLink(newFile, file); + } + + return super.visitFile(file, attrs); + } + } + ); } /**