Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DF3FF18585 for ; Thu, 11 Jun 2015 17:13:05 +0000 (UTC) Received: (qmail 3530 invoked by uid 500); 11 Jun 2015 17:13:05 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 3461 invoked by uid 500); 11 Jun 2015 17:13:05 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 3447 invoked by uid 99); 11 Jun 2015 17:13:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Jun 2015 17:13:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7D989DFFD5; Thu, 11 Jun 2015 17:13:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wang@apache.org To: common-commits@hadoop.apache.org Message-Id: <39ad757ea9a64946a4734ea6d54af0fe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline. Contributed by Eddy Xu. Date: Thu, 11 Jun 2015 17:13:05 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 7454ed2b5 -> 086eb4942 HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline. Contributed by Eddy Xu. (cherry picked from commit b258b344bb76af6492828201959e36b45f0f75b8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/086eb494 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/086eb494 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/086eb494 Branch: refs/heads/branch-2 Commit: 086eb49424eb1befe0d1be4dbef509d16e41b0f1 Parents: 7454ed2 Author: Andrew Wang Authored: Thu Jun 11 10:12:31 2015 -0700 Committer: Andrew Wang Committed: Thu Jun 11 10:12:40 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/BlockReceiver.java | 10 +--------- .../hdfs/server/datanode/ReplicaInPipeline.java | 15 ++++++++++++++- .../server/datanode/ReplicaInPipelineInterface.java | 10 ++++++++++ .../hdfs/server/datanode/SimulatedFSDataset.java | 5 +++++ .../extdataset/ExternalReplicaInPipeline.java | 7 +++++++ 6 files changed, 40 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/086eb494/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8ae4fe3..8d6054c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -280,6 +280,9 @@ Release 2.8.0 - UNRELEASED HDFS-8549. Abort the balancer if an upgrade is in progress. (wang) + HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to + ReplicaInPipeline. (Eddy Xu via wang) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/086eb494/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 2e11600..c46892d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -23,7 +23,6 @@ import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; import java.io.FileDescriptor; import java.io.FileOutputStream; import java.io.IOException; @@ -839,15 +838,8 @@ class BlockReceiver implements Closeable { // In case this datanode is shutting down for quick restart, // send a special ack upstream. if (datanode.isRestarting() && isClient && !isTransfer) { - File blockFile = ((ReplicaInPipeline)replicaInfo).getBlockFile(); - File restartMeta = new File(blockFile.getParent() + - File.pathSeparator + "." + blockFile.getName() + ".restart"); - if (restartMeta.exists() && !restartMeta.delete()) { - LOG.warn("Failed to delete restart meta file: " + - restartMeta.getPath()); - } try (Writer out = new OutputStreamWriter( - new FileOutputStream(restartMeta), "UTF-8")) { + replicaInfo.createRestartMetaStream(), "UTF-8")) { // write out the current time. out.write(Long.toString(Time.now() + restartBudget)); out.flush(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/086eb494/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java index 0eb143a..d9406f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.RandomAccessFile; import org.apache.hadoop.hdfs.protocol.Block; @@ -287,7 +288,19 @@ public class ReplicaInPipeline extends ReplicaInfo throw e; } } - + + @Override + public OutputStream createRestartMetaStream() throws IOException { + File blockFile = getBlockFile(); + File restartMeta = new File(blockFile.getParent() + + File.pathSeparator + "." + blockFile.getName() + ".restart"); + if (restartMeta.exists() && !restartMeta.delete()) { + DataNode.LOG.warn("Failed to delete restart meta file: " + + restartMeta.getPath()); + } + return new FileOutputStream(restartMeta); + } + @Override public String toString() { return super.toString() http://git-wip-us.apache.org/repos/asf/hadoop/blob/086eb494/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java index 0263d0f..ef9f3e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.IOException; +import java.io.OutputStream; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.util.DataChecksum; @@ -73,4 +74,13 @@ public interface ReplicaInPipelineInterface extends Replica { */ public ReplicaOutputStreams createStreams(boolean isCreate, DataChecksum requestedChecksum) throws IOException; + + /** + * Create an output stream to write restart metadata in case of datanode + * shutting down for quick restart. + * + * @return output stream for writing. + * @throws IOException if any error occurs + */ + public OutputStream createRestartMetaStream() throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/086eb494/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 778dd28..6e5f07c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -263,6 +263,11 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override + public OutputStream createRestartMetaStream() throws IOException { + return new SimulatedOutputStream(); + } + + @Override synchronized public long getBlockId() { return theBlock.getBlockId(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/086eb494/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java index ad44500..a0039bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.datanode.extdataset; import java.io.IOException; +import java.io.OutputStream; + import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum; import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; @@ -60,6 +62,11 @@ public class ExternalReplicaInPipeline implements ReplicaInPipelineInterface { } @Override + public OutputStream createRestartMetaStream() throws IOException { + return null; + } + + @Override public long getBlockId() { return 0; }