hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [11/50] [abbrv] hadoop git commit: HDFS-11877. FileJournalManager#getLogFile should ignore in progress edit logs during JN sync. Contributed by Hanisha Koneru.
Date Wed, 31 May 2017 22:42:46 GMT
HDFS-11877. FileJournalManager#getLogFile should ignore in progress edit logs during JN sync.
Contributed by Hanisha Koneru.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e83ed5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e83ed5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e83ed5e

Branch: refs/heads/YARN-1011
Commit: 0e83ed5e7372c801c9fee01df91b6b56de467ab1
Parents: 1c8dd6d
Author: Arpit Agarwal <arp@apache.org>
Authored: Wed May 24 16:09:00 2017 -0700
Committer: Arpit Agarwal <arp@apache.org>
Committed: Wed May 24 16:09:00 2017 -0700

----------------------------------------------------------------------
 .../hdfs/qjournal/client/IPCLoggerChannel.java    |  2 +-
 .../qjournal/server/GetJournalEditServlet.java    | 16 +++++++++++++---
 .../hdfs/qjournal/server/JournalNodeSyncer.java   |  2 +-
 .../hdfs/server/namenode/FileJournalManager.java  | 18 +++++++++++++++---
 4 files changed, 30 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index cbfe5b8..6cd892c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -276,7 +276,7 @@ public class IPCLoggerChannel implements AsyncLogger {
         
     try {
       String path = GetJournalEditServlet.buildPath(
-          journalId, segmentTxId, nsInfo);
+          journalId, segmentTxId, nsInfo, true);
       return new URL(httpServerURL, path);
     } catch (MalformedURLException e) {
       // should never get here.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
index 2335ea5..e96fd4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
@@ -72,6 +72,7 @@ public class GetJournalEditServlet extends HttpServlet {
   static final String STORAGEINFO_PARAM = "storageInfo";
   static final String JOURNAL_ID_PARAM = "jid";
   static final String SEGMENT_TXID_PARAM = "segmentTxId";
+  static final String IN_PROGRESS_OK = "inProgressOk";
 
   protected boolean isValidRequestor(HttpServletRequest request, Configuration conf)
       throws IOException {
@@ -186,6 +187,14 @@ public class GetJournalEditServlet extends HttpServlet {
       final Configuration conf = (Configuration) getServletContext()
           .getAttribute(JspHelper.CURRENT_CONF);
       final String journalId = request.getParameter(JOURNAL_ID_PARAM);
+      final String inProgressOkStr = request.getParameter(IN_PROGRESS_OK);
+      final boolean inProgressOk;
+      if (inProgressOkStr != null &&
+          inProgressOkStr.equalsIgnoreCase("false")) {
+        inProgressOk = false;
+      } else {
+        inProgressOk = true;
+      }
       QuorumJournalManager.checkJournalId(journalId);
       final JNStorage storage = JournalNodeHttpServer
           .getJournalFromContext(context, journalId).getStorage();
@@ -210,8 +219,7 @@ public class GetJournalEditServlet extends HttpServlet {
         // Synchronize on the FJM so that the file doesn't get finalized
         // out from underneath us while we're in the process of opening
         // it up.
-        EditLogFile elf = fjm.getLogFile(
-            segmentTxId);
+        EditLogFile elf = fjm.getLogFile(segmentTxId, inProgressOk);
         if (elf == null) {
           response.sendError(HttpServletResponse.SC_NOT_FOUND,
               "No edit log found starting at txid " + segmentTxId);
@@ -239,7 +247,7 @@ public class GetJournalEditServlet extends HttpServlet {
   }
 
   public static String buildPath(String journalId, long segmentTxId,
-      NamespaceInfo nsInfo) {
+      NamespaceInfo nsInfo, boolean inProgressOk) {
     StringBuilder path = new StringBuilder("/getJournal?");
     try {
       path.append(JOURNAL_ID_PARAM).append("=")
@@ -248,6 +256,8 @@ public class GetJournalEditServlet extends HttpServlet {
           .append(segmentTxId);
       path.append("&" + STORAGEINFO_PARAM).append("=")
           .append(URLEncoder.encode(nsInfo.toColonSeparatedString(), "UTF-8"));
+      path.append("&" + IN_PROGRESS_OK).append("=")
+          .append(inProgressOk);
     } catch (UnsupportedEncodingException e) {
       // Never get here -- everyone supports UTF-8
       throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
index 788c5de..73defc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
@@ -296,7 +296,7 @@ public class JournalNodeSyncer {
           }
 
           String urlPath = GetJournalEditServlet.buildPath(jid, missingLog
-              .getStartTxId(), nsInfo);
+              .getStartTxId(), nsInfo, false);
           url = new URL(remoteJNproxy.httpServerUrl, urlPath);
           success = downloadMissingLogSegment(url, missingLog);
         } catch (MalformedURLException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e83ed5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
index 5d62825..c71c09a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
@@ -451,16 +451,28 @@ public class FileJournalManager implements JournalManager {
   }
   
   public EditLogFile getLogFile(long startTxId) throws IOException {
-    return getLogFile(sd.getCurrentDir(), startTxId);
+    return getLogFile(sd.getCurrentDir(), startTxId, true);
   }
-  
+
+  public EditLogFile getLogFile(long startTxId, boolean inProgressOk)
+      throws IOException {
+    return getLogFile(sd.getCurrentDir(), startTxId, inProgressOk);
+  }
+
   public static EditLogFile getLogFile(File dir, long startTxId)
       throws IOException {
+    return getLogFile(dir, startTxId, true);
+  }
+
+  public static EditLogFile getLogFile(File dir, long startTxId,
+      boolean inProgressOk) throws IOException {
     List<EditLogFile> files = matchEditLogs(dir);
     List<EditLogFile> ret = Lists.newLinkedList();
     for (EditLogFile elf : files) {
       if (elf.getFirstTxId() == startTxId) {
-        ret.add(elf);
+        if (inProgressOk || !elf.isInProgress()) {
+          ret.add(elf);
+        }
       }
     }
     


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message