hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject svn commit: r1502420 - in /hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/security/token/delegation/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/java/org/apache/hadoop/...
Date Fri, 12 Jul 2013 04:41:06 GMT
Author: cnauroth
Date: Fri Jul 12 04:41:05 2013
New Revision: 1502420

URL: http://svn.apache.org/r1502420
Log:
HDFS-4372. Merging change r1502419 from branch-2 to branch-2.1-beta.

Added:
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/
      - copied from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/AbstractTracking.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/AbstractTracking.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Phase.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Phase.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/PhaseTracking.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/PhaseTracking.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressMetrics.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressMetrics.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Status.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Status.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StepTracking.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StepTracking.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StepType.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StepType.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/package-info.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/package-info.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/
      - copied from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressTestHelper.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressTestHelper.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgressMetrics.java
      - copied unchanged from r1502419, hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgressMetrics.java
Modified:
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
    hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jul 12 04:41:05 2013
@@ -70,6 +70,8 @@ Release 2.1.0-beta - 2013-07-02
 
     HDFS-4762 Provide HDFS based NFSv3 and Mountd implementation (brandonli)
 
+    HDFS-4372. Track NameNode startup progress. (cnauroth)
+    
   IMPROVEMENTS
 
     HDFS-4461. DirectoryScanner: volume path prefix takes up memory for every

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Fri Jul 12 04:41:05 2013
@@ -31,6 +31,11 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.security.Credentials;
@@ -127,14 +132,15 @@ public class DelegationTokenSecretManage
    * Store the current state of the SecretManager for persistence
    * 
    * @param out Output stream for writing into fsimage.
+   * @param sdPath String storage directory path
    * @throws IOException
    */
-  public synchronized void saveSecretManagerState(DataOutputStream out)
-      throws IOException {
+  public synchronized void saveSecretManagerState(DataOutputStream out,
+      String sdPath) throws IOException {
     out.writeInt(currentId);
-    saveAllKeys(out);
+    saveAllKeys(out, sdPath);
     out.writeInt(delegationTokenSequenceNumber);
-    saveCurrentTokens(out);
+    saveCurrentTokens(out, sdPath);
   }
   
   /**
@@ -237,8 +243,13 @@ public class DelegationTokenSecretManage
   /**
    * Private helper methods to save delegation keys and tokens in fsimage
    */
-  private synchronized void saveCurrentTokens(DataOutputStream out)
-      throws IOException {
+  private synchronized void saveCurrentTokens(DataOutputStream out,
+      String sdPath) throws IOException {
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = new Step(StepType.DELEGATION_TOKENS, sdPath);
+    prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+    prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
+    Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
     out.writeInt(currentTokens.size());
     Iterator<DelegationTokenIdentifier> iter = currentTokens.keySet()
         .iterator();
@@ -247,20 +258,29 @@ public class DelegationTokenSecretManage
       id.write(out);
       DelegationTokenInformation info = currentTokens.get(id);
       out.writeLong(info.getRenewDate());
+      counter.increment();
     }
+    prog.endStep(Phase.SAVING_CHECKPOINT, step);
   }
   
   /*
    * Save the current state of allKeys
    */
-  private synchronized void saveAllKeys(DataOutputStream out)
+  private synchronized void saveAllKeys(DataOutputStream out, String sdPath)
       throws IOException {
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = new Step(StepType.DELEGATION_KEYS, sdPath);
+    prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+    prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
+    Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
     out.writeInt(allKeys.size());
     Iterator<Integer> iter = allKeys.keySet().iterator();
     while (iter.hasNext()) {
       Integer key = iter.next();
       allKeys.get(key).write(out);
+      counter.increment();
     }
+    prog.endStep(Phase.SAVING_CHECKPOINT, step);
   }
   
   /**
@@ -268,13 +288,20 @@ public class DelegationTokenSecretManage
    */
   private synchronized void loadCurrentTokens(DataInput in)
       throws IOException {
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = new Step(StepType.DELEGATION_TOKENS);
+    prog.beginStep(Phase.LOADING_FSIMAGE, step);
     int numberOfTokens = in.readInt();
+    prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfTokens);
+    Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
     for (int i = 0; i < numberOfTokens; i++) {
       DelegationTokenIdentifier id = new DelegationTokenIdentifier();
       id.readFields(in);
       long expiryTime = in.readLong();
       addPersistedDelegationToken(id, expiryTime);
+      counter.increment();
     }
+    prog.endStep(Phase.LOADING_FSIMAGE, step);
   }
 
   /**
@@ -283,12 +310,19 @@ public class DelegationTokenSecretManage
    * @throws IOException
    */
   private synchronized void loadAllKeys(DataInput in) throws IOException {
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = new Step(StepType.DELEGATION_KEYS);
+    prog.beginStep(Phase.LOADING_FSIMAGE, step);
     int numberOfKeys = in.readInt();
+    prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfKeys);
+    Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
     for (int i = 0; i < numberOfKeys; i++) {
       DelegationKey value = new DelegationKey();
       value.readFields(in);
       addKey(value);
+      counter.increment();
     }
+    prog.endStep(Phase.LOADING_FSIMAGE, step);
   }
 
   /**

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java Fri Jul 12 04:41:05 2013
@@ -407,8 +407,6 @@ public class EditLogFileInputStream exte
 
     @Override
     public long length() {
-      Preconditions.checkState(advertisedSize != -1,
-          "must get input stream before length is available");
       return advertisedSize;
     }
 

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java Fri Jul 12 04:41:05 2013
@@ -33,7 +33,18 @@ import java.io.IOException;
 @InterfaceStability.Evolving
 public abstract class EditLogInputStream implements Closeable {
   private FSEditLogOp cachedOp = null; 
-  
+
+  /**
+   * Returns the name of the currently active underlying stream.  The default
+   * implementation returns the same value as getName unless overridden by the
+   * subclass.
+   * 
+   * @return String name of the currently active underlying stream
+   */
+  public String getCurrentStreamName() {
+    return getName();
+  }
+
   /** 
    * @return the name of the EditLogInputStream
    */
@@ -157,7 +168,9 @@ public abstract class EditLogInputStream
   public abstract long getPosition();
 
   /**
-   * Return the size of the current edits log.
+   * Return the size of the current edits log or -1 if unknown.
+   * 
+   * @return long size of the current edits log or -1 if unknown
    */
   public abstract long length() throws IOException;
   

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Fri Jul 12 04:41:05 2013
@@ -71,7 +71,12 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampV2Op;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
 import org.apache.hadoop.hdfs.util.Holder;
+import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.base.Joiner;
 
@@ -95,6 +100,9 @@ public class FSEditLogLoader {
    */
   long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId,
       MetaRecoveryContext recovery) throws IOException {
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = createStartupProgressStep(edits);
+    prog.beginStep(Phase.LOADING_EDITS, step);
     fsNamesys.writeLock();
     try {
       long startTime = now();
@@ -108,6 +116,7 @@ public class FSEditLogLoader {
     } finally {
       edits.close();
       fsNamesys.writeUnlock();
+      prog.endStep(Phase.LOADING_EDITS, step);
     }
   }
 
@@ -133,6 +142,10 @@ public class FSEditLogLoader {
     long numEdits = 0;
     long lastTxId = in.getLastTxId();
     long numTxns = (lastTxId - expectedStartingTxId) + 1;
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = createStartupProgressStep(in);
+    prog.setTotal(Phase.LOADING_EDITS, step, numTxns);
+    Counter counter = prog.getCounter(Phase.LOADING_EDITS, step);
     long lastLogTime = now();
     long lastInodeId = fsNamesys.getLastInodeId();
     
@@ -193,7 +206,7 @@ public class FSEditLogLoader {
           }
           // Now that the operation has been successfully decoded and
           // applied, update our bookkeeping.
-          incrOpCount(op.opCode, opCounts);
+          incrOpCount(op.opCode, opCounts, step, counter);
           if (op.hasTransactionId()) {
             lastAppliedTxId = op.getTransactionId();
             expectedTxId = lastAppliedTxId + 1;
@@ -684,7 +697,8 @@ public class FSEditLogLoader {
   }
 
   private void incrOpCount(FSEditLogOpCodes opCode,
-      EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts) {
+      EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts, Step step,
+      Counter counter) {
     Holder<Integer> holder = opCounts.get(opCode);
     if (holder == null) {
       holder = new Holder<Integer>(1);
@@ -692,6 +706,7 @@ public class FSEditLogLoader {
     } else {
       holder.held++;
     }
+    counter.increment();
   }
 
   /**
@@ -863,4 +878,20 @@ public class FSEditLogLoader {
   public long getLastAppliedTxId() {
     return lastAppliedTxId;
   }
+
+  /**
+   * Creates a Step used for updating startup progress, populated with
+   * information from the given edits.  The step always includes the log's name.
+   * If the log has a known length, then the length is included in the step too.
+   * 
+   * @param edits EditLogInputStream to use for populating step
+   * @return Step populated with information from edits
+   * @throws IOException thrown if there is an I/O error
+   */
+  private static Step createStartupProgressStep(EditLogInputStream edits)
+      throws IOException {
+    long length = edits.length();
+    String name = edits.getCurrentStreamName();
+    return length != -1 ? new Step(name, length) : new Step(name);
+  }
 }

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Fri Jul 12 04:41:05 2013
@@ -56,6 +56,8 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
 import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
@@ -65,6 +67,7 @@ import org.apache.hadoop.hdfs.util.Cance
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.util.IdGenerator;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -587,6 +590,12 @@ public class FSImage implements Closeabl
     isUpgradeFinalized = inspector.isUpgradeFinalized();
  
     List<FSImageFile> imageFiles = inspector.getLatestImages();
+
+    StartupProgress prog = NameNode.getStartupProgress();
+    prog.beginPhase(Phase.LOADING_FSIMAGE);
+    File phaseFile = imageFiles.get(0).getFile();
+    prog.setFile(Phase.LOADING_FSIMAGE, phaseFile.getAbsolutePath());
+    prog.setSize(Phase.LOADING_FSIMAGE, phaseFile.length());
     boolean needToSave = inspector.needToSave();
 
     Iterable<EditLogInputStream> editStreams = null;
@@ -636,6 +645,7 @@ public class FSImage implements Closeabl
       FSEditLog.closeAllStreams(editStreams);
       throw new IOException("Failed to load an FSImage file!");
     }
+    prog.endPhase(Phase.LOADING_FSIMAGE);
     long txnsAdvanced = loadEdits(editStreams, target, recovery);
     needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
                                                     txnsAdvanced);
@@ -710,6 +720,8 @@ public class FSImage implements Closeabl
   public long loadEdits(Iterable<EditLogInputStream> editStreams,
       FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
     LOG.debug("About to load edits:\n  " + Joiner.on("\n  ").join(editStreams));
+    StartupProgress prog = NameNode.getStartupProgress();
+    prog.beginPhase(Phase.LOADING_EDITS);
     
     long prevLastAppliedTxId = lastAppliedTxId;  
     try {    
@@ -736,6 +748,7 @@ public class FSImage implements Closeabl
       // update the counts
       updateCountForQuota(target.dir.rootDir);   
     }
+    prog.endPhase(Phase.LOADING_EDITS);
     return lastAppliedTxId - prevLastAppliedTxId;
   }
 
@@ -949,6 +962,8 @@ public class FSImage implements Closeabl
   protected synchronized void saveFSImageInAllDirs(FSNamesystem source, long txid,
       Canceler canceler)
       throws IOException {    
+    StartupProgress prog = NameNode.getStartupProgress();
+    prog.beginPhase(Phase.SAVING_CHECKPOINT);
     if (storage.getNumStorageDirs(NameNodeDirType.IMAGE) == 0) {
       throw new IOException("No image directories available!");
     }
@@ -994,6 +1009,7 @@ public class FSImage implements Closeabl
       ctx.markComplete();
       ctx = null;
     }
+    prog.endPhase(Phase.SAVING_CHECKPOINT);
   }
 
   /**

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Fri Jul 12 04:41:05 2013
@@ -58,6 +58,11 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.io.Text;
@@ -233,6 +238,9 @@ public class FSImageFormat {
       checkNotLoaded();
       assert curFile != null : "curFile is null";
 
+      StartupProgress prog = NameNode.getStartupProgress();
+      Step step = new Step(StepType.INODES);
+      prog.beginStep(Phase.LOADING_FSIMAGE, step);
       long startTime = now();
 
       //
@@ -322,18 +330,24 @@ public class FSImageFormat {
         
         // load all inodes
         LOG.info("Number of files = " + numFiles);
+        prog.setTotal(Phase.LOADING_FSIMAGE, step, numFiles);
+        Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
         if (LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION,
             imgVersion)) {
           if (supportSnapshot) {
-            loadLocalNameINodesWithSnapshot(in);
+            loadLocalNameINodesWithSnapshot(numFiles, in, counter);
           } else {
-            loadLocalNameINodes(numFiles, in);
+            loadLocalNameINodes(numFiles, in, counter);
           }
         } else {
-          loadFullNameINodes(numFiles, in);
+          loadFullNameINodes(numFiles, in, counter);
         }
 
-        loadFilesUnderConstruction(in, supportSnapshot);
+        loadFilesUnderConstruction(in, supportSnapshot, counter);
+        prog.endStep(Phase.LOADING_FSIMAGE, step);
+        // Now that the step is finished, set counter equal to total to adjust
+        // for possible under-counting due to reference inodes.
+        prog.setCount(Phase.LOADING_FSIMAGE, step, numFiles);
 
         loadSecretManagerState(in);
 
@@ -367,18 +381,20 @@ public class FSImageFormat {
      * Load fsimage files when 1) only local names are stored, 
      * and 2) snapshot is supported.
      * 
+     * @param numFiles number of files expected to be read
      * @param in Image input stream
+     * @param counter Counter to increment for namenode startup progress
      */
-    private void loadLocalNameINodesWithSnapshot(DataInput in)
-        throws IOException {
+    private void loadLocalNameINodesWithSnapshot(long numFiles, DataInput in,
+        Counter counter) throws IOException {
       assert LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION,
           getLayoutVersion());
       assert LayoutVersion.supports(Feature.SNAPSHOT, getLayoutVersion());
       
       // load root
-      loadRoot(in);
+      loadRoot(in, counter);
       // load rest of the nodes recursively
-      loadDirectoryWithSnapshot(in);
+      loadDirectoryWithSnapshot(in, counter);
     }
     
   /** 
@@ -386,22 +402,23 @@ public class FSImageFormat {
    *   
    * @param numFiles number of files expected to be read
    * @param in image input stream
+   * @param counter Counter to increment for namenode startup progress
    * @throws IOException
    */  
-   private void loadLocalNameINodes(long numFiles, DataInput in) 
+   private void loadLocalNameINodes(long numFiles, DataInput in, Counter counter)
        throws IOException {
      assert LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION,
          getLayoutVersion());
      assert numFiles > 0;
 
      // load root
-     loadRoot(in);
+     loadRoot(in, counter);
      // have loaded the first file (the root)
      numFiles--; 
 
      // load rest of the nodes directory by directory
      while (numFiles > 0) {
-       numFiles -= loadDirectory(in);
+       numFiles -= loadDirectory(in, counter);
      }
      if (numFiles != 0) {
        throw new IOException("Read unexpect number of files: " + -numFiles);
@@ -412,24 +429,27 @@ public class FSImageFormat {
      * Load information about root, and use the information to update the root
      * directory of NameSystem.
      * @param in The {@link DataInput} instance to read.
+     * @param counter Counter to increment for namenode startup progress
      */
-    private void loadRoot(DataInput in) throws IOException {
+    private void loadRoot(DataInput in, Counter counter)
+        throws IOException {
       // load root
       if (in.readShort() != 0) {
         throw new IOException("First node is not root");
       }
-      final INodeDirectory root = loadINode(null, false, in).asDirectory();
+      final INodeDirectory root = loadINode(null, false, in, counter)
+        .asDirectory();
       // update the root's attributes
       updateRootAttr(root);
     }
    
     /** Load children nodes for the parent directory. */
-    private int loadChildren(INodeDirectory parent, DataInput in)
-        throws IOException {
+    private int loadChildren(INodeDirectory parent, DataInput in,
+        Counter counter) throws IOException {
       int numChildren = in.readInt();
       for (int i = 0; i < numChildren; i++) {
         // load single inode
-        INode newNode = loadINodeWithLocalName(false, in, true);
+        INode newNode = loadINodeWithLocalName(false, in, true, counter);
         addToParent(parent, newNode);
       }
       return numChildren;
@@ -438,8 +458,9 @@ public class FSImageFormat {
     /**
      * Load a directory when snapshot is supported.
      * @param in The {@link DataInput} instance to read.
+     * @param counter Counter to increment for namenode startup progress
      */
-    private void loadDirectoryWithSnapshot(DataInput in)
+    private void loadDirectoryWithSnapshot(DataInput in, Counter counter)
         throws IOException {
       // Step 1. Identify the parent INode
       long inodeId = in.readLong();
@@ -470,7 +491,7 @@ public class FSImageFormat {
       }
 
       // Step 3. Load children nodes under parent
-      loadChildren(parent, in);
+      loadChildren(parent, in, counter);
       
       // Step 4. load Directory Diff List
       SnapshotFSImageFormat.loadDirectoryDiffList(parent, in, this);
@@ -479,7 +500,7 @@ public class FSImageFormat {
       // directories
       int numSubTree = in.readInt();
       for (int i = 0; i < numSubTree; i++) {
-        loadDirectoryWithSnapshot(in);
+        loadDirectoryWithSnapshot(in, counter);
       }
     }
     
@@ -487,14 +508,15 @@ public class FSImageFormat {
     * Load all children of a directory
     * 
     * @param in
+    * @param counter Counter to increment for namenode startup progress
     * @return number of child inodes read
     * @throws IOException
     */
-   private int loadDirectory(DataInput in) throws IOException {
+   private int loadDirectory(DataInput in, Counter counter) throws IOException {
      String parentPath = FSImageSerialization.readString(in);
      final INodeDirectory parent = INodeDirectory.valueOf(
          namesystem.dir.rootDir.getNode(parentPath, true), parentPath);
-     return loadChildren(parent, in);
+     return loadChildren(parent, in, counter);
    }
 
   /**
@@ -502,10 +524,11 @@ public class FSImageFormat {
    * 
    * @param numFiles total number of files to load
    * @param in data input stream
+   * @param counter Counter to increment for namenode startup progress
    * @throws IOException if any error occurs
    */
-  private void loadFullNameINodes(long numFiles,
-      DataInput in) throws IOException {
+  private void loadFullNameINodes(long numFiles, DataInput in, Counter counter)
+      throws IOException {
     byte[][] pathComponents;
     byte[][] parentPath = {{}};      
     FSDirectory fsDir = namesystem.dir;
@@ -513,7 +536,7 @@ public class FSImageFormat {
     for (long i = 0; i < numFiles; i++) {
       pathComponents = FSImageSerialization.readPathComponents(in);
       final INode newNode = loadINode(
-          pathComponents[pathComponents.length-1], false, in);
+          pathComponents[pathComponents.length-1], false, in, counter);
 
       if (isRoot(pathComponents)) { // it is the root
         // update the root's attributes
@@ -580,10 +603,16 @@ public class FSImageFormat {
       return namesystem.dir;
     }
 
+    public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in,
+        boolean updateINodeMap) throws IOException {
+      return loadINodeWithLocalName(isSnapshotINode, in, updateINodeMap, null);
+    }
+
     public INode loadINodeWithLocalName(boolean isSnapshotINode,
-        DataInput in, boolean updateINodeMap) throws IOException {
+        DataInput in, boolean updateINodeMap, Counter counter)
+        throws IOException {
       final byte[] localName = FSImageSerialization.readLocalName(in);
-      INode inode = loadINode(localName, isSnapshotINode, in);
+      INode inode = loadINode(localName, isSnapshotINode, in, counter);
       if (updateINodeMap
           && LayoutVersion.supports(Feature.ADD_INODE_ID, getLayoutVersion())) {
         namesystem.dir.addToInodeMap(inode);
@@ -595,10 +624,11 @@ public class FSImageFormat {
    * load an inode from fsimage except for its name
    * 
    * @param in data input stream from which image is read
+   * @param counter Counter to increment for namenode startup progress
    * @return an inode
    */
   INode loadINode(final byte[] localName, boolean isSnapshotINode,
-      DataInput in) throws IOException {
+      DataInput in, Counter counter) throws IOException {
     final int imgVersion = getLayoutVersion();
     if (LayoutVersion.supports(Feature.SNAPSHOT, imgVersion)) {
       namesystem.getFSDirectory().verifyINodeName(localName);
@@ -650,6 +680,9 @@ public class FSImageFormat {
       final PermissionStatus permissions = PermissionStatus.read(in);
 
       // return
+      if (counter != null) {
+        counter.increment();
+      }
       final INodeFile file = new INodeFile(inodeId, localName, permissions,
           modificationTime, atime, blocks, replication, blockSize);
       return fileDiffs != null? new INodeFileWithSnapshot(file, fileDiffs)
@@ -679,6 +712,9 @@ public class FSImageFormat {
       final PermissionStatus permissions = PermissionStatus.read(in);
 
       //return
+      if (counter != null) {
+        counter.increment();
+      }
       final INodeDirectory dir = nsQuota >= 0 || dsQuota >= 0?
           new INodeDirectoryWithQuota(inodeId, localName, permissions,
               modificationTime, nsQuota, dsQuota)
@@ -691,10 +727,16 @@ public class FSImageFormat {
 
       final String symlink = Text.readString(in);
       final PermissionStatus permissions = PermissionStatus.read(in);
+      if (counter != null) {
+        counter.increment();
+      }
       return new INodeSymlink(inodeId, localName, permissions,
           modificationTime, atime, symlink);
     } else if (numBlocks == -3) {
       //reference
+      // Intentionally do not increment counter, because it is too difficult at
+      // this point to assess whether or not this is a reference that counts
+      // toward quota.
       
       final boolean isWithName = in.readBoolean();
       // lastSnapshotId for WithName node, dstSnapshotId for DstReference node
@@ -761,7 +803,7 @@ public class FSImageFormat {
     }
   
     private void loadFilesUnderConstruction(DataInput in,
-        boolean supportSnapshot) throws IOException {
+        boolean supportSnapshot, Counter counter) throws IOException {
       FSDirectory fsDir = namesystem.dir;
       int size = in.readInt();
 
@@ -770,6 +812,7 @@ public class FSImageFormat {
       for (int i = 0; i < size; i++) {
         INodeFileUnderConstruction cons = FSImageSerialization
             .readINodeUnderConstruction(in, namesystem, getLayoutVersion());
+        counter.increment();
 
         // verify that file exists in namespace
         String path = cons.getLocalName();
@@ -888,6 +931,13 @@ public class FSImageFormat {
 
       final FSNamesystem sourceNamesystem = context.getSourceNamesystem();
       FSDirectory fsDir = sourceNamesystem.dir;
+      String sdPath = newFile.getParentFile().getParentFile().getAbsolutePath();
+      Step step = new Step(StepType.INODES, sdPath);
+      StartupProgress prog = NameNode.getStartupProgress();
+      prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+      prog.setTotal(Phase.SAVING_CHECKPOINT, step,
+        fsDir.rootDir.numItemsInTree());
+      Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
       long startTime = now();
       //
       // Write out data
@@ -922,14 +972,18 @@ public class FSImageFormat {
                  " using " + compression);
 
         // save the root
-        FSImageSerialization.saveINode2Image(fsDir.rootDir, out, false,
-            referenceMap);
+        saveINode2Image(fsDir.rootDir, out, false, referenceMap, counter);
         // save the rest of the nodes
-        saveImage(fsDir.rootDir, out, true);
+        saveImage(fsDir.rootDir, out, true, counter);
+        prog.endStep(Phase.SAVING_CHECKPOINT, step);
+        // Now that the step is finished, set counter equal to total to adjust
+        // for possible under-counting due to reference inodes.
+        prog.setCount(Phase.SAVING_CHECKPOINT, step,
+          fsDir.rootDir.numItemsInTree());
         // save files under construction
         sourceNamesystem.saveFilesUnderConstruction(out);
         context.checkCancelled();
-        sourceNamesystem.saveSecretManagerState(out);
+        sourceNamesystem.saveSecretManagerState(out, sdPath);
         context.checkCancelled();
         out.flush();
         context.checkCancelled();
@@ -950,17 +1004,18 @@ public class FSImageFormat {
      * Save children INodes.
      * @param children The list of children INodes
      * @param out The DataOutputStream to write
+     * @param counter Counter to increment for namenode startup progress
      * @return Number of children that are directory
      */
-    private int saveChildren(ReadOnlyList<INode> children, DataOutputStream out)
-        throws IOException {
+    private int saveChildren(ReadOnlyList<INode> children, DataOutputStream out,
+        Counter counter) throws IOException {
       // Write normal children INode. 
       out.writeInt(children.size());
       int dirNum = 0;
       int i = 0;
       for(INode child : children) {
         // print all children first
-        FSImageSerialization.saveINode2Image(child, out, false, referenceMap);
+        saveINode2Image(child, out, false, referenceMap, counter);
         if (child.isDirectory()) {
           dirNum++;
         }
@@ -983,9 +1038,10 @@ public class FSImageFormat {
      * @param toSaveSubtree Whether or not to save the subtree to fsimage. For
      *                      reference node, its subtree may already have been
      *                      saved before.
+     * @param counter Counter to increment for namenode startup progress
      */
     private void saveImage(INodeDirectory current, DataOutputStream out,
-        boolean toSaveSubtree) throws IOException {
+        boolean toSaveSubtree, Counter counter) throws IOException {
       // write the inode id of the directory
       out.writeLong(current.getId());
       
@@ -1014,7 +1070,7 @@ public class FSImageFormat {
       }
 
       // 3. Write children INode 
-      dirNum += saveChildren(children, out);
+      dirNum += saveChildren(children, out, counter);
       
       // 4. Write DirectoryDiff lists, if there is any.
       SnapshotFSImageFormat.saveDirectoryDiffList(current, out, referenceMap);
@@ -1029,16 +1085,39 @@ public class FSImageFormat {
         // make sure we only save the subtree under a reference node once
         boolean toSave = child.isReference() ? 
             referenceMap.toProcessSubtree(child.getId()) : true;
-        saveImage(child.asDirectory(), out, toSave);
+        saveImage(child.asDirectory(), out, toSave, counter);
       }
       if (snapshotDirs != null) {
         for (INodeDirectory subDir : snapshotDirs) {
           // make sure we only save the subtree under a reference node once
           boolean toSave = subDir.getParentReference() != null ? 
               referenceMap.toProcessSubtree(subDir.getId()) : true;
-          saveImage(subDir, out, toSave);
+          saveImage(subDir, out, toSave, counter);
         }
       }
     }
+
+    /**
+     * Saves inode and increments progress counter.
+     * 
+     * @param inode INode to save
+     * @param out DataOutputStream to receive inode
+     * @param writeUnderConstruction boolean true if this is under construction
+     * @param referenceMap ReferenceMap containing reference inodes
+     * @param counter Counter to increment for namenode startup progress
+     * @throws IOException thrown if there is an I/O error
+     */
+    private void saveINode2Image(INode inode, DataOutputStream out,
+        boolean writeUnderConstruction, ReferenceMap referenceMap,
+        Counter counter) throws IOException {
+      FSImageSerialization.saveINode2Image(inode, out, writeUnderConstruction,
+        referenceMap);
+      // Intentionally do not increment counter for reference inodes, because it
+      // is too difficult at this point to assess whether or not this is a
+      // reference that counts toward quota.
+      if (!(inode instanceof INodeReference)) {
+        counter.increment();
+      }
+    }
   }
 }

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Jul 12 04:41:05 2013
@@ -168,6 +168,12 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
 import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
 import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
@@ -311,7 +317,10 @@ public class FSNamesystem implements Nam
     TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
   final DelegationTokenSecretManager dtSecretManager;
   private final boolean alwaysUseDelegationTokensForTests;
-  
+
+  private static final Step STEP_AWAITING_REPORTED_BLOCKS =
+    new Step(StepType.AWAITING_REPORTED_BLOCKS);
+
   // Tracks whether the default audit logger is the only configured audit
   // logger; this allows isAuditEnabled() to return false in case the
   // underlying logger is disabled, and avoid some unnecessary work.
@@ -699,8 +708,15 @@ public class FSNamesystem implements Nam
     try {
       // We shouldn't be calling saveNamespace if we've come up in standby state.
       MetaRecoveryContext recovery = startOpt.createRecoveryContext();
-      if (fsImage.recoverTransitionRead(startOpt, this, recovery) && !haEnabled) {
+      boolean needToSave =
+        fsImage.recoverTransitionRead(startOpt, this, recovery) && !haEnabled;
+      if (needToSave) {
         fsImage.saveNamespace(this);
+      } else {
+        // No need to save, so mark the phase done.
+        StartupProgress prog = NameNode.getStartupProgress();
+        prog.beginPhase(Phase.SAVING_CHECKPOINT);
+        prog.endPhase(Phase.SAVING_CHECKPOINT);
       }
       // This will start a new log segment and write to the seen_txid file, so
       // we shouldn't do it when coming up in standby state
@@ -759,6 +775,10 @@ public class FSNamesystem implements Nam
       checkAvailableResources();
       assert safeMode != null &&
         !safeMode.isPopulatingReplQueues();
+      StartupProgress prog = NameNode.getStartupProgress();
+      prog.beginPhase(Phase.SAFEMODE);
+      prog.setTotal(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS,
+        getCompleteBlocksTotal());
       setBlockTotal();
       blockManager.activate(conf);
     } finally {
@@ -4058,6 +4078,8 @@ public class FSNamesystem implements Nam
     private boolean resourcesLow = false;
     /** Should safemode adjust its block totals as blocks come in */
     private boolean shouldIncrementallyTrackBlocks = false;
+    /** counter for tracking startup progress of reported blocks */
+    private Counter awaitingReportedBlocksCounter;
     
     /**
      * Creates SafeModeInfo when the name node enters
@@ -4178,6 +4200,13 @@ public class FSNamesystem implements Nam
           + blockManager.numOfUnderReplicatedBlocks() + " blocks");
 
       startSecretManagerIfNecessary();
+
+      // If startup has not yet completed, end safemode phase.
+      StartupProgress prog = NameNode.getStartupProgress();
+      if (prog.getStatus(Phase.SAFEMODE) != Status.COMPLETE) {
+        prog.endStep(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS);
+        prog.endPhase(Phase.SAFEMODE);
+      }
     }
 
     /**
@@ -4295,6 +4324,17 @@ public class FSNamesystem implements Nam
     private synchronized void incrementSafeBlockCount(short replication) {
       if (replication == safeReplication) {
         this.blockSafe++;
+
+        // Report startup progress only if we haven't completed startup yet.
+        StartupProgress prog = NameNode.getStartupProgress();
+        if (prog.getStatus(Phase.SAFEMODE) != Status.COMPLETE) {
+          if (this.awaitingReportedBlocksCounter == null) {
+            this.awaitingReportedBlocksCounter = prog.getCounter(Phase.SAFEMODE,
+              STEP_AWAITING_REPORTED_BLOCKS);
+          }
+          this.awaitingReportedBlocksCounter.increment();
+        }
+
         checkMode();
       }
     }
@@ -5630,9 +5670,11 @@ public class FSNamesystem implements Nam
   
   /**
    * @param out save state of the secret manager
+   * @param sdPath String storage directory path
    */
-  void saveSecretManagerState(DataOutputStream out) throws IOException {
-    dtSecretManager.saveSecretManagerState(out);
+  void saveSecretManagerState(DataOutputStream out, String sdPath)
+      throws IOException {
+    dtSecretManager.saveSecretManagerState(out, sdPath);
   }
 
   /**

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Jul 12 04:41:05 2013
@@ -59,6 +59,8 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
 import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressMetrics;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
@@ -262,6 +264,10 @@ public class NameNode {
   }
 
   static NameNodeMetrics metrics;
+  private static final StartupProgress startupProgress = new StartupProgress();
+  static {
+    StartupProgressMetrics.register(startupProgress);
+  }
 
   /** Return the {@link FSNamesystem} object.
    * @return {@link FSNamesystem} object.
@@ -281,7 +287,16 @@ public class NameNode {
   public static NameNodeMetrics getNameNodeMetrics() {
     return metrics;
   }
-  
+
+  /**
+   * Returns object used for reporting namenode startup progress.
+   * 
+   * @return StartupProgress for reporting namenode startup progress
+   */
+  public static StartupProgress getStartupProgress() {
+    return startupProgress;
+  }
+
   public static InetSocketAddress getAddress(String address) {
     return NetUtils.createSocketAddr(address, DEFAULT_PORT);
   }
@@ -434,15 +449,19 @@ public class NameNode {
     loginAsNameNodeUser(conf);
 
     NameNode.initMetrics(conf, this.getRole());
+
+    if (NamenodeRole.NAMENODE == role) {
+      startHttpServer(conf);
+      validateConfigurationSettingsOrAbort(conf);
+    }
     loadNamesystem(conf);
 
     rpcServer = createRpcServer(conf);
-    
-    try {
-      validateConfigurationSettings(conf);
-    } catch (IOException e) {
-      LOG.fatal(e.toString());
-      throw e;
+    if (NamenodeRole.NAMENODE == role) {
+      httpServer.setNameNodeAddress(getNameNodeAddress());
+      httpServer.setFSImage(getFSImage());
+    } else {
+      validateConfigurationSettingsOrAbort(conf);
     }
 
     startCommonServices(conf);
@@ -479,10 +498,31 @@ public class NameNode {
     } 
   }
 
+  /**
+   * Validate NameNode configuration.  Log a fatal error and abort if
+   * configuration is invalid.
+   * 
+   * @param conf Configuration to validate
+   * @throws IOException thrown if conf is invalid
+   */
+  private void validateConfigurationSettingsOrAbort(Configuration conf)
+      throws IOException {
+    try {
+      validateConfigurationSettings(conf);
+    } catch (IOException e) {
+      LOG.fatal(e.toString());
+      throw e;
+    }
+  }
+
   /** Start the services common to active and standby states */
   private void startCommonServices(Configuration conf) throws IOException {
     namesystem.startCommonServices(conf, haContext);
-    startHttpServer(conf);
+    if (NamenodeRole.NAMENODE != role) {
+      startHttpServer(conf);
+      httpServer.setNameNodeAddress(getNameNodeAddress());
+      httpServer.setFSImage(getFSImage());
+    }
     rpcServer.start();
     plugins = conf.getInstances(DFS_NAMENODE_PLUGINS_KEY,
         ServicePlugin.class);
@@ -550,6 +590,7 @@ public class NameNode {
   private void startHttpServer(final Configuration conf) throws IOException {
     httpServer = new NameNodeHttpServer(conf, this, getHttpServerAddress(conf));
     httpServer.start();
+    httpServer.setStartupProgress(startupProgress);
     setHttpServerAddress(conf);
   }
   

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Fri Jul 12 04:41:05 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
 import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
 import org.apache.hadoop.hdfs.web.AuthFilter;
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
@@ -55,6 +56,7 @@ public class NameNodeHttpServer {
   public static final String NAMENODE_ADDRESS_ATTRIBUTE_KEY = "name.node.address";
   public static final String FSIMAGE_ATTRIBUTE_KEY = "name.system.image";
   protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node";
+  public static final String STARTUP_PROGRESS_ATTRIBUTE_KEY = "startup.progress";
   
   public NameNodeHttpServer(
       Configuration conf,
@@ -146,9 +148,6 @@ public class NameNodeHttpServer {
         .getPort());
     }
     httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
-    httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY,
-        NetUtils.getConnectAddress(nn.getNameNodeAddress()));
-    httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, nn.getFSImage());
     httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
     setupServlets(httpServer, conf);
     httpServer.start();
@@ -166,6 +165,34 @@ public class NameNodeHttpServer {
     return httpAddress;
   }
 
+  /**
+   * Sets fsimage for use by servlets.
+   * 
+   * @param fsImage FSImage to set
+   */
+  public void setFSImage(FSImage fsImage) {
+    httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, fsImage);
+  }
+
+  /**
+   * Sets address of namenode for use by servlets.
+   * 
+   * @param nameNodeAddress InetSocketAddress to set
+   */
+  public void setNameNodeAddress(InetSocketAddress nameNodeAddress) {
+    httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY,
+        NetUtils.getConnectAddress(nameNodeAddress));
+  }
+
+  /**
+   * Sets startup progress of namenode for use by servlets.
+   * 
+   * @param prog StartupProgress to set
+   */
+  public void setStartupProgress(StartupProgress prog) {
+    httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog);
+  }
+
   private static void setupServlets(HttpServer httpServer, Configuration conf) {
     httpServer.addInternalServlet("getDelegationToken",
         GetDelegationTokenServlet.PATH_SPEC, 
@@ -207,4 +234,15 @@ public class NameNodeHttpServer {
     return (InetSocketAddress)context.getAttribute(
         NAMENODE_ADDRESS_ATTRIBUTE_KEY);
   }
+
+  /**
+   * Returns StartupProgress associated with ServletContext.
+   * 
+   * @param context ServletContext to get
+   * @return StartupProgress associated with context
+   */
+  public static StartupProgress getStartupProgressFromContext(
+      ServletContext context) {
+    return (StartupProgress)context.getAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY);
+  }
 }

Modified: hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java?rev=1502420&r1=1502419&r2=1502420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java Fri Jul 12 04:41:05 2013
@@ -123,6 +123,11 @@ class RedundantEditLogInputStream extend
   }
 
   @Override
+  public String getCurrentStreamName() {
+    return streams[curIdx].getCurrentStreamName();
+  }
+
+  @Override
   public String getName() {
     StringBuilder bld = new StringBuilder();
     String prefix = "";



Mime
View raw message