hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject hadoop git commit: HDFS-11209. SNN can't checkpoint when rolling upgrade is not finalized. Contributed by Xiaoyu Yao.
Date Tue, 17 Jan 2017 22:45:02 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 3017069a7 -> 420e4db98


HDFS-11209. SNN can't checkpoint when rolling upgrade is not finalized. Contributed by Xiaoyu
Yao.

(cherry picked from commit b1fce2b8b14c4ce43f7098a269ac2b95304db0ce)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/420e4db9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/420e4db9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/420e4db9

Branch: refs/heads/branch-2
Commit: 420e4db98a1cf5864e40109a24576846ca30e5b2
Parents: 3017069
Author: Xiaoyu Yao <xyao@apache.org>
Authored: Tue Jan 17 14:33:26 2017 -0800
Committer: Xiaoyu Yao <xyao@apache.org>
Committed: Tue Jan 17 14:36:03 2017 -0800

----------------------------------------------------------------------
 .../NamenodeProtocolServerSideTranslatorPB.java     | 16 ++++++++++++++++
 .../protocolPB/NamenodeProtocolTranslatorPB.java    | 15 +++++++++++++++
 .../hadoop/hdfs/server/namenode/Checkpointer.java   |  4 ++--
 .../apache/hadoop/hdfs/server/namenode/FSImage.java |  4 ++--
 .../hdfs/server/namenode/NameNodeRpcServer.java     |  7 +++++++
 .../hdfs/server/namenode/SecondaryNameNode.java     |  6 +++---
 .../hdfs/server/protocol/NamenodeProtocol.java      |  8 ++++++++
 .../src/main/proto/NamenodeProtocol.proto           | 16 ++++++++++++++++
 8 files changed, 69 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index c7b3e74..6a10fe4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecen
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
@@ -240,4 +242,18 @@ public class NamenodeProtocolServerSideTranslatorPB implements
     return IsUpgradeFinalizedResponseProto.newBuilder()
         .setIsUpgradeFinalized(isUpgradeFinalized).build();
   }
+
+  @Override
+  public IsRollingUpgradeResponseProto isRollingUpgrade(
+      RpcController controller, IsRollingUpgradeRequestProto request)
+      throws ServiceException {
+    boolean isRollingUpgrade;
+    try {
+      isRollingUpgrade = impl.isRollingUpgrade();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    return IsRollingUpgradeResponseProto.newBuilder()
+        .setIsRollingUpgrade(isRollingUpgrade).build();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 6fc5fc7..02074f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlocksReq
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetEditLogManifestRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
@@ -247,4 +249,17 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
       throw ProtobufHelper.getRemoteException(e);
     }
   }
+
+  @Override
+  public boolean isRollingUpgrade() throws IOException {
+    IsRollingUpgradeRequestProto req = IsRollingUpgradeRequestProto
+        .newBuilder().build();
+    try {
+      IsRollingUpgradeResponseProto response = rpcProxy.isRollingUpgrade(
+          NULL_CONTROLLER, req);
+      return response.getIsRollingUpgrade();
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
index 9ff40cc..318acfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
@@ -260,8 +260,8 @@ class Checkpointer extends Daemon {
             completeBlocksTotal);
       }
       bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid);
-      if (!backupNode.namesystem.isRollingUpgrade()) {
-        bnStorage.writeAll();
+      if (!backupNode.namenode.isRollingUpgrade()) {
+        bnImage.updateStorageVersion();
       }
     } finally {
       backupNode.namesystem.writeUnlock("doCheckpoint");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 916f6ce..1e93784 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -538,7 +538,7 @@ public class FSImage implements Closeable {
 
     // and save it but keep the same checkpointTime
     saveNamespace(target);
-    getStorage().writeAll();
+    updateStorageVersion();
   }
   
   void finalizeUpgrade(boolean finalizeEditLog) throws IOException {
@@ -1031,7 +1031,7 @@ public class FSImage implements Closeable {
       try {
         saveFSImageInAllDirs(source, nnf, imageTxId, canceler);
         if (!source.isRollingUpgrade()) {
-          storage.writeAll();
+          updateStorageVersion();
         }
       } finally {
         if (editLogWasOpen) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index e14406e..8a113b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -1244,6 +1244,13 @@ public class NameNodeRpcServer implements NamenodeProtocols {
     namesystem.checkSuperuserPrivilege();
     return namesystem.isUpgradeFinalized();
   }
+
+  @Override // NamenodeProtocol
+  public boolean isRollingUpgrade() throws IOException {
+    checkNNStartup();
+    namesystem.checkSuperuserPrivilege();
+    return namesystem.isRollingUpgrade();
+  }
     
   @Override // ClientProtocol
   public void finalizeUpgrade() throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
index acb2c8a..1358f46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
@@ -1064,7 +1064,7 @@ public class SecondaryNameNode implements Runnable,
 
   }
     
-  static void doMerge(
+  void doMerge(
       CheckpointSignature sig, RemoteEditLogManifest manifest,
       boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
       throws IOException {   
@@ -1093,8 +1093,8 @@ public class SecondaryNameNode implements Runnable,
     Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
     // The following has the side effect of purging old fsimages/edit logs.
     dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId());
-    if (!dstNamesystem.isRollingUpgrade()) {
-      dstStorage.writeAll();
+    if (!namenode.isRollingUpgrade()) {
+      dstImage.updateStorageVersion();
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
index 4048372..ccdf516c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
@@ -193,5 +193,13 @@ public interface NamenodeProtocol {
   @Idempotent
   public boolean isUpgradeFinalized() throws IOException;
 
+  /**
+   * return whether the Namenode is rolling upgrade in progress (true) or
+   * not (false).
+   * @return
+   * @throws IOException
+   */
+  @Idempotent
+  boolean isRollingUpgrade() throws IOException;
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/420e4db9/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
index d8b1e44..8aa09d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
@@ -193,6 +193,16 @@ message IsUpgradeFinalizedResponseProto {
 }
 
 /**
+ * void request
+ */
+message IsRollingUpgradeRequestProto {
+}
+
+message IsRollingUpgradeResponseProto {
+  required bool isRollingUpgrade = 1;
+}
+
+/**
  * Protocol used by the sub-ordinate namenode to send requests
  * the active/primary namenode.
  *
@@ -267,4 +277,10 @@ service NamenodeProtocolService {
    */
   rpc isUpgradeFinalized(IsUpgradeFinalizedRequestProto)
       returns (IsUpgradeFinalizedResponseProto);
+
+  /**
+   * Return whether the NameNode is in rolling upgrade (true) or not (false).
+   */
+  rpc isRollingUpgrade(IsRollingUpgradeRequestProto)
+      returns (IsRollingUpgradeResponseProto);
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message