Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 84DF1172B9 for ; Tue, 1 Sep 2015 23:59:20 +0000 (UTC) Received: (qmail 8237 invoked by uid 500); 1 Sep 2015 23:59:20 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 8167 invoked by uid 500); 1 Sep 2015 23:59:20 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 8158 invoked by uid 99); 1 Sep 2015 23:59:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Sep 2015 23:59:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 29BA7E0606; Tue, 1 Sep 2015 23:59:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinodkv@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-7929. inotify unable fetch pre-upgrade edit log segments once upgrade starts (Zhe Zhang via Colin P. McCabe) Date: Tue, 1 Sep 2015 23:59:20 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.6.1 49ef26b5a -> 81a445edf HDFS-7929. inotify unable fetch pre-upgrade edit log segments once upgrade starts (Zhe Zhang via Colin P. McCabe) (cherry picked from commit 43b41f22411439c5e23629197fb2fde45dcf0f0f) (cherry picked from commit 219eb22c1571f76df32967a930049d983cbf5024) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java (cherry picked from commit 03798416bfe27383c52e4d9f632fe9fa168c6e95) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81a445ed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81a445ed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81a445ed Branch: refs/heads/branch-2.6.1 Commit: 81a445edf81f42c90a05d764dfebfadfafad622b Parents: 49ef26b Author: Colin Patrick Mccabe Authored: Wed Mar 18 18:48:54 2015 -0700 Committer: Vinod Kumar Vavilapalli Committed: Tue Sep 1 16:55:50 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/namenode/FSImage.java | 2 +- .../server/namenode/FileJournalManager.java | 2 +- .../hdfs/server/namenode/NNUpgradeUtil.java | 44 ++++++++++++++++-- .../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 48 +++++++++++++++++++- 5 files changed, 93 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 44a7139..ff335ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -97,6 +97,9 @@ Release 2.6.1 - UNRELEASED HDFS-7587. Edit log corruption can happen if append fails with a quota violation. (jing9) + HDFS-7929. inotify unable fetch pre-upgrade edit log segments once upgrade + starts (Zhe Zhang via Colin P. McCabe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 9b72421..51efb51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -393,7 +393,7 @@ public class FSImage implements Closeable { for (Iterator it = storage.dirIterator(false); it.hasNext();) { StorageDirectory sd = it.next(); try { - NNUpgradeUtil.doPreUpgrade(sd); + NNUpgradeUtil.doPreUpgrade(conf, sd); } catch (Exception e) { LOG.error("Failed to move aside pre-upgrade storage " + "in image directory " + sd.getRoot(), e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index 101c42c..2df052b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -585,7 +585,7 @@ public class FileJournalManager implements JournalManager { public void doPreUpgrade() throws IOException { LOG.info("Starting upgrade of edits directory " + sd.getRoot()); try { - NNUpgradeUtil.doPreUpgrade(sd); + NNUpgradeUtil.doPreUpgrade(conf, sd); } catch (IOException ioe) { LOG.error("Failed to move aside pre-upgrade storage " + "in image directory " + sd.getRoot(), ioe); http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java index 546480d..c63da20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java @@ -18,10 +18,13 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; @@ -99,15 +102,17 @@ abstract class NNUpgradeUtil { * a call to any JM's or local storage dir's doPreUpgrade method fails, then * doUpgrade will not be called for any JM. The existing current dir is * renamed to previous.tmp, and then a new, empty current dir is created. - * + * + * @param conf configuration for creating {@link EditLogFileOutputStream} * @param sd the storage directory to perform the pre-upgrade procedure. * @throws IOException in the event of error */ - static void doPreUpgrade(StorageDirectory sd) throws IOException { + static void doPreUpgrade(Configuration conf, StorageDirectory sd) + throws IOException { LOG.info("Starting upgrade of storage directory " + sd.getRoot()); File curDir = sd.getCurrentDir(); File prevDir = sd.getPreviousDir(); - File tmpDir = sd.getPreviousTmp(); + final File tmpDir = sd.getPreviousTmp(); Preconditions.checkState(curDir.exists(), "Current directory must exist for preupgrade."); @@ -123,6 +128,39 @@ abstract class NNUpgradeUtil { if (!curDir.mkdir()) { throw new IOException("Cannot create directory " + curDir); } + + String[] fileNameList = tmpDir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return dir.equals(tmpDir) + && name.startsWith(NNStorage.NameNodeFile.EDITS.getName()); + } + }); + if (fileNameList == null) { + throw new IOException("either " + tmpDir + + " is not a directory or there is an I/O error"); + } + + for (String s : fileNameList) { + File prevFile = new File(tmpDir, s); + Preconditions.checkState(prevFile.canRead(), + "Edits log file " + s + " is not readable."); + File newFile = new File(curDir, prevFile.getName()); + Preconditions.checkState(newFile.createNewFile(), + "Cannot create new edits log file in " + curDir); + EditLogFileInputStream in = new EditLogFileInputStream(prevFile); + EditLogFileOutputStream out = + new EditLogFileOutputStream(conf, newFile, 512*1024); + FSEditLogOp logOp = in.nextValidOp(); + while (logOp != null) { + out.write(logOp); + logOp = in.nextOp(); + } + out.setReadyToFlush(); + out.flushAndSync(true); + out.close(); + in.close(); + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/81a445ed/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java index bb00144..49762ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java @@ -28,7 +28,11 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import java.util.regex.Pattern; import org.apache.commons.logging.Log; @@ -42,6 +46,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StringUtils; @@ -450,7 +455,48 @@ public class TestDFSUpgrade { assertTrue(Storage.is203LayoutVersion(lv)); } } - + + @Test + public void testPreserveEditLogs() throws Exception { + conf = new HdfsConfiguration(); + conf = UpgradeUtilities.initializeStorageStateConf(1, conf); + String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY); + conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false); + + log("Normal NameNode upgrade", 1); + File[] created = + UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); + List beforeUpgrade = new LinkedList(); + for (final File createdDir : created) { + String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE); + Collections.addAll(beforeUpgrade, fileNameList); + } + + cluster = createCluster(); + + List afterUpgrade = new LinkedList(); + for (final File createdDir : created) { + String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE); + Collections.addAll(afterUpgrade, fileNameList); + } + + for (String s : beforeUpgrade) { + assertTrue(afterUpgrade.contains(s)); + } + + cluster.shutdown(); + UpgradeUtilities.createEmptyDirs(nameNodeDirs); + } + + private static enum EditLogsFilter implements FilenameFilter { + INSTANCE; + + @Override + public boolean accept(File dir, String name) { + return name.startsWith(NNStorage.NameNodeFile.EDITS.getName()); + } + } + public static void main(String[] args) throws Exception { TestDFSUpgrade t = new TestDFSUpgrade(); TestDFSUpgrade.initialize();