hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1062045 [1/2] - in /hadoop/hdfs/trunk: ./ bin/ src/docs/src/documentation/content/xdocs/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/server/protocol/ src/java/org/apache/hadoop/hdfs/tools/offlineEditsVi...
Date Fri, 21 Jan 2011 22:08:05 GMT
Author: szetszwo
Date: Fri Jan 21 22:08:04 2011
New Revision: 1062045

URL: http://svn.apache.org/viewvc?rev=1062045&view=rev
Log:
HDFS-1448. Add a new tool Offline Edits Viewer (oev).  Contributed by Erik Steffl

Added:
    hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TextEditsVisitor.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/Tokenizer.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TokenizerFactory.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlTokenizer.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml
Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/bin/hdfs
    hadoop/hdfs/trunk/build.xml
    hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Jan 21 22:08:04 2011
@@ -14,6 +14,9 @@ Trunk (unreleased changes)
     HDFS-1482. Add listCorruptFileBlocks to DistributedFileSystem.
     (Patrick Kling via hairong)
 
+    HDFS-1448. Add a new tool Offline Edits Viewer (oev).  (Erik Steffl
+    via szetszwo)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)

Modified: hadoop/hdfs/trunk/bin/hdfs
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/bin/hdfs?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/bin/hdfs (original)
+++ hadoop/hdfs/trunk/bin/hdfs Fri Jan 21 22:08:04 2011
@@ -32,6 +32,7 @@ function print_usage(){
   echo "  balancer             run a cluster balancing utility"
   echo "  jmxget               get JMX exported values from NameNode or DataNode."
   echo "  oiv                  apply the offline fsimage viewer to an fsimage"
+  echo "  oev                  apply the offline edits viewer to an edits file"
   echo "  fetchdt              fetch a delegation token from the NameNode"
   echo "						Use -help to see options"
   echo ""
@@ -89,6 +90,8 @@ elif [ "$COMMAND" = "jmxget" ] ; then
   CLASS=org.apache.hadoop.hdfs.tools.JMXGet
 elif [ "$COMMAND" = "oiv" ] ; then
   CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
+elif [ "$COMMAND" = "oev" ] ; then
+  CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
 elif [ "$COMMAND" = "fetchdt" ] ; then
   CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
 else

Modified: hadoop/hdfs/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/build.xml?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/build.xml (original)
+++ hadoop/hdfs/trunk/build.xml Fri Jan 21 22:08:04 2011
@@ -422,6 +422,8 @@
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data1k" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml" todir="${test.cache.data}"/>
   </target>
 
   <macrodef name="macro-compile-hdfs-test">

Added: hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml (added)
+++ hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml Fri Jan 21 22:08:04 2011
@@ -0,0 +1,157 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+
+  <header>
+    <title>Offline Edits Viewer Guide</title>
+    <authors>
+      <person name="Erik Steffl" email="steffl@yahoo-inc.com"/>
+    </authors> 
+  </header>
+
+  <body>
+
+    <section>
+
+      <title>Overview</title>
+
+      <p>
+        Offline Edits Viewer is a tool to parse the Edits log file. The
+        current processors are mostly useful for conversion between
+        different formats, including XML which is human readable and
+        easier to edit than native binary format.
+      </p>
+
+      <p>
+        The tool can parse the edits formats -18 (roughly Hadoop 0.19)
+        and later. The tool operates on files only, it does not need
+        Hadoop cluster to be running.
+      </p>
+
+      <p>Input formats supported:</p>
+      <ol>
+        <li><strong>binary</strong>: native binary format that Hadoop uses internally</li>
+        <li>
+          <strong>xml</strong>: XML format, as produced by
+          <strong>xml</strong> processor, used if filename has xml
+          (case insensitive) extension
+        </li>
+      </ol>
+
+      <p>
+        The Offline Edits Viewer provides several output processors
+        (unless stated otherwise the output of the processor can be
+        converted back to original edits file):
+      </p>
+      <ol>
+        <li><strong>binary</strong>: native binary format that Hadoop uses internally</li>
+        <li><strong>xml</strong>: XML format</li>
+        <li><strong>stats</strong>: prints out statistics, this cannot be converted back to Edits file</li>
+      </ol>
+
+    </section> <!-- Overview -->
+
+    <section>
+
+      <title>Usage</title>
+
+      <p><code>bash$ bin/hdfs oev -i edits -o edits.xml</code></p>
+
+      <table>
+        <tr><th>Flag</th><th>Description</th></tr>
+        <tr>
+          <td><code>[-i|--inputFile] &lt;input file&gt;</code></td>
+          <td>
+            Specify the input edits log file to process. Xml (case
+            insensitive) extension means XML format otherwise binary
+            format is assumed. Required.
+          </td>
+        </tr>
+        <tr>
+          <td><code>[-o|--outputFile] &lt;output file&gt;</code></td>
+          <td>
+            Specify the output filename, if the specified output processor
+            generates one. If the specified file already exists, it is
+            silently overwritten. Required.
+          </td>
+        </tr>
+        <tr>
+          <td><code>[-p|--processor] &lt;processor&gt;</code></td>
+          <td>
+            Specify the image processor to apply against the image
+            file. Currently valid options are <strong>binary</strong>,
+            <strong>xml</strong> (default) and <strong>stats</strong>.
+          </td>
+        </tr>
+        <tr>
+          <td><code>[-v|--verbose]-</code></td>
+          <td>
+            Print the input and output filenames and pipe output of
+            processor to console as well as specified file. On extremely
+            large files, this may increase processing time by an order
+            of magnitude.
+          </td>
+        </tr>
+        <tr>
+          <td><code>[-h|--help]</code></td>
+          <td>
+            Display the tool usage and help information and exit.
+          </td>
+        </tr>
+      </table>
+
+    </section> <!-- Usage -->
+
+    <section>
+
+      <title>Case study: Hadoop cluster recovery</title>
+
+      <p>
+        In case there is some problem with hadoop cluster and the edits
+        file is corrupted it is possible to save at least part of the
+        edits file that is correct. This can be done by converting the
+        binary edits to XML, edit it manually and then convert it back
+        to binary. The most common problem is that the edits file is
+        missing the closing record (record that has opCode -1). This
+        should be recognized by the tool and the XML format should be
+        properly closed.
+      </p>
+
+      <p>
+        If there is no closing record in the XML file you can add one
+        after last correct record. Anything after the record with opCode
+        -1 is ignored.
+      </p>
+
+      <p>Example of a closing record (with opCode -1):</p>
+<source>
+&lt;RECORD&gt;
+  &lt;OPCODE&gt;-1&lt;/OPCODE&gt;
+  &lt;DATA&gt;
+  &lt;/DATA&gt;
+&lt;/RECORD&gt;
+</source>
+
+    </section> <!-- Case study: Hadoop cluster recovery -->
+
+  </body>
+
+</document>

Modified: hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml Fri Jan 21 22:08:04 2011
@@ -42,6 +42,7 @@ See http://forrest.apache.org/docs/linki
       <hdfs_quotas     	    label="Quotas" href="hdfs_quota_admin_guide.html" />
       <hdfs_SLG        		label="Synthetic Load Generator"  href="SLG_user_guide.html" />
       <hdfs_imageviewer	label="Offline Image Viewer"	href="hdfs_imageviewer.html" />
+      <hdfs_editsviewer	label="Offline Edits Viewer"	href="hdfs_editsviewer.html" />
       <hdfsproxy 			    label="HDFS Proxy" href="hdfsproxy.html"/>
       <faultinject_framework label="Fault Injection"  href="faultinject_framework.html" /> 
       <hdfs_libhdfs   		label="C API libhdfs" href="libhdfs.html" /> 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java Fri Jan 21 22:08:04 2011
@@ -147,7 +147,7 @@ class EditLogBackupOutputStream extends 
       JournalRecord jRec = null;
       for(; idx < bufReadySize; idx++) {
         jRec = bufReady.get(idx);
-        if(jRec.op >= FSEditLog.Ops.OP_JSPOOL_START)
+        if(jRec.op >= FSEditLogOpCodes.OP_JSPOOL_START.getOpCode())
           break;  // special operation should be sent in a separate call to BN
         jRec.write(out);
       }
@@ -177,7 +177,7 @@ class EditLogBackupOutputStream extends 
   private void send(int ja) throws IOException {
     try {
       int length = out.getLength();
-      out.write(FSEditLog.Ops.OP_INVALID);
+      out.write(FSEditLogOpCodes.OP_INVALID.getOpCode());
       backupNode.journal(nnRegistration, ja, length, out.getData());
     } finally {
       out.reset();

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Fri Jan 21 22:08:04 2011
@@ -128,7 +128,7 @@ class EditLogFileOutputStream extends Ed
   @Override
   void setReadyToFlush() throws IOException {
     assert bufReady.size() == 0 : "previous data is not flushed yet";
-    write(FSEditLog.Ops.OP_INVALID); // insert end-of-file marker
+    write(FSEditLogOpCodes.OP_INVALID.getOpCode()); // insert eof marker
     DataOutputBuffer tmp = bufReady;
     bufReady = bufCurrent;
     bufCurrent = tmp;
@@ -189,7 +189,7 @@ class EditLogFileOutputStream extends Ed
    */
   @Override
   boolean isOperationSupported(byte op) {
-    return op < FSEditLog.Ops.OP_JSPOOL_START - 1;
+    return op < FSEditLogOpCodes.OP_JSPOOL_START.getOpCode() - 1;
   }
 
   /**

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri Jan 21 22:08:04 2011
@@ -48,6 +48,8 @@ import org.apache.hadoop.io.LongWritable
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
 
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*;
+
 /**
  * FSEditLog maintains a log of the namespace modifications.
  * 
@@ -55,44 +57,6 @@ import org.apache.hadoop.security.token.
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class FSEditLog {
-  
-  abstract static class Ops {
-    public  static final byte OP_INVALID = -1;
-    public static final byte OP_ADD = 0;
-    public static final byte OP_RENAME_OLD = 1;  // rename
-    public static final byte OP_DELETE = 2;  // delete
-    public static final byte OP_MKDIR = 3;   // create directory
-    public static final byte OP_SET_REPLICATION = 4; // set replication
-    //the following two are used only for backward compatibility :
-    @Deprecated public static final byte OP_DATANODE_ADD = 5;
-    @Deprecated public static final byte OP_DATANODE_REMOVE = 6;
-    public static final byte OP_SET_PERMISSIONS = 7;
-    public static final byte OP_SET_OWNER = 8;
-    public static final byte OP_CLOSE = 9;    // close after write
-    public static final byte OP_SET_GENSTAMP = 10;    // store genstamp
-    /* The following two are not used any more. Should be removed once
-     * LAST_UPGRADABLE_LAYOUT_VERSION is -17 or newer. */
-    public static final byte OP_SET_NS_QUOTA = 11; // set namespace quota
-    public static final byte OP_CLEAR_NS_QUOTA = 12; // clear namespace quota
-    public static final byte OP_TIMES = 13; // sets mod & access time on a file
-    public static final byte OP_SET_QUOTA = 14; // sets name and disk quotas.
-    public static final byte OP_RENAME = 15;  // new rename
-    public static final byte OP_CONCAT_DELETE = 16; // concat files.
-    public static final byte OP_SYMLINK = 17; // a symbolic link
-    public static final byte OP_GET_DELEGATION_TOKEN = 18; //new delegation token
-    public static final byte OP_RENEW_DELEGATION_TOKEN = 19; //renew delegation token
-    public static final byte OP_CANCEL_DELEGATION_TOKEN = 20; //cancel delegation token
-    public static final byte OP_UPDATE_MASTER_KEY = 21; //update master key
-  
-    /* 
-     * The following operations are used to control remote edit log streams,
-     * and not logged into file streams.
-     */
-    static final byte OP_JSPOOL_START = // start journal spool
-                                      NamenodeProtocol.JA_JSPOOL_START;
-    static final byte OP_CHECKPOINT_TIME = // incr checkpoint time
-                                      NamenodeProtocol.JA_CHECKPOINT_TIME;
-  }
 
   static final String NO_JOURNAL_STREAMS_WARNING = "!!! WARNING !!!" +
       " File system changes are not persistent. No journal streams.";
@@ -370,7 +334,7 @@ public class FSEditLog {
    * Write an operation to the edit log. Do not sync to persistent
    * store yet.
    */
-  void logEdit(byte op, Writable ... writables) {
+  void logEdit(FSEditLogOpCodes opCode, Writable ... writables) {
     synchronized (this) {
       // wait if an automatic sync is scheduled
       waitIfAutoSyncScheduled();
@@ -380,10 +344,10 @@ public class FSEditLog {
       ArrayList<EditLogOutputStream> errorStreams = null;
       long start = now();
       for(EditLogOutputStream eStream : editStreams) {
-        if(!eStream.isOperationSupported(op))
+        if(!eStream.isOperationSupported(opCode.getOpCode()))
           continue;
         try {
-          eStream.write(op, writables);
+          eStream.write(opCode.getOpCode(), writables);
         } catch (IOException ie) {
           LOG.error("logEdit: removing "+ eStream.getName(), ie);
           if(errorStreams == null)
@@ -643,7 +607,7 @@ public class FSEditLog {
       FSEditLog.toLogLong(newNode.getModificationTime()),
       FSEditLog.toLogLong(newNode.getAccessTime()),
       FSEditLog.toLogLong(newNode.getPreferredBlockSize())};
-    logEdit(Ops.OP_ADD,
+    logEdit(OP_ADD,
             new ArrayWritable(DeprecatedUTF8.class, nameReplicationPair), 
             new ArrayWritable(Block.class, newNode.getBlocks()),
             newNode.getPermissionStatus(),
@@ -661,7 +625,7 @@ public class FSEditLog {
       FSEditLog.toLogLong(newNode.getModificationTime()),
       FSEditLog.toLogLong(newNode.getAccessTime()),
       FSEditLog.toLogLong(newNode.getPreferredBlockSize())};
-    logEdit(Ops.OP_CLOSE,
+    logEdit(OP_CLOSE,
             new ArrayWritable(DeprecatedUTF8.class, nameReplicationPair),
             new ArrayWritable(Block.class, newNode.getBlocks()),
             newNode.getPermissionStatus());
@@ -676,8 +640,9 @@ public class FSEditLog {
       FSEditLog.toLogLong(newNode.getModificationTime()),
       FSEditLog.toLogLong(newNode.getAccessTime())
     };
-    logEdit(Ops.OP_MKDIR, new ArrayWritable(DeprecatedUTF8.class, info),
-        newNode.getPermissionStatus());
+    logEdit(OP_MKDIR,
+      new ArrayWritable(DeprecatedUTF8.class, info),
+      newNode.getPermissionStatus());
   }
   
   /** 
@@ -689,7 +654,7 @@ public class FSEditLog {
       new DeprecatedUTF8(src),
       new DeprecatedUTF8(dst),
       FSEditLog.toLogLong(timestamp)};
-    logEdit(Ops.OP_RENAME_OLD, new ArrayWritable(DeprecatedUTF8.class, info));
+    logEdit(OP_RENAME_OLD, new ArrayWritable(DeprecatedUTF8.class, info));
   }
   
   /** 
@@ -700,17 +665,18 @@ public class FSEditLog {
       new DeprecatedUTF8(src),
       new DeprecatedUTF8(dst),
       FSEditLog.toLogLong(timestamp)};
-    logEdit(Ops.OP_RENAME, new ArrayWritable(DeprecatedUTF8.class, info),
-        toBytesWritable(options));
+    logEdit(OP_RENAME,
+      new ArrayWritable(DeprecatedUTF8.class, info),
+      toBytesWritable(options));
   }
   
   /** 
    * Add set replication record to edit log
    */
   void logSetReplication(String src, short replication) {
-    logEdit(Ops.OP_SET_REPLICATION, 
-            new DeprecatedUTF8(src), 
-            FSEditLog.toLogReplication(replication));
+    logEdit(OP_SET_REPLICATION, 
+      new DeprecatedUTF8(src), 
+      FSEditLog.toLogReplication(replication));
   }
   
   /** Add set namespace quota record to edit log
@@ -719,20 +685,21 @@ public class FSEditLog {
    * @param quota the directory size limit
    */
   void logSetQuota(String src, long nsQuota, long dsQuota) {
-    logEdit(Ops.OP_SET_QUOTA, new DeprecatedUTF8(src), 
-            new LongWritable(nsQuota), new LongWritable(dsQuota));
+    logEdit(OP_SET_QUOTA,
+      new DeprecatedUTF8(src), 
+      new LongWritable(nsQuota), new LongWritable(dsQuota));
   }
 
   /**  Add set permissions record to edit log */
   void logSetPermissions(String src, FsPermission permissions) {
-    logEdit(Ops.OP_SET_PERMISSIONS, new DeprecatedUTF8(src), permissions);
+    logEdit(OP_SET_PERMISSIONS, new DeprecatedUTF8(src), permissions);
   }
 
   /**  Add set owner record to edit log */
   void logSetOwner(String src, String username, String groupname) {
     DeprecatedUTF8 u = new DeprecatedUTF8(username == null? "": username);
     DeprecatedUTF8 g = new DeprecatedUTF8(groupname == null? "": groupname);
-    logEdit(Ops.OP_SET_OWNER, new DeprecatedUTF8(src), u, g);
+    logEdit(OP_SET_OWNER, new DeprecatedUTF8(src), u, g);
   }
   
   /**
@@ -747,7 +714,7 @@ public class FSEditLog {
       info[idx++] = new DeprecatedUTF8(srcs[i]);
     }
     info[idx] = FSEditLog.toLogLong(timestamp);
-    logEdit(Ops.OP_CONCAT_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
+    logEdit(OP_CONCAT_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
   }
   
   /** 
@@ -757,14 +724,14 @@ public class FSEditLog {
     DeprecatedUTF8 info[] = new DeprecatedUTF8[] { 
       new DeprecatedUTF8(src),
       FSEditLog.toLogLong(timestamp)};
-    logEdit(Ops.OP_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
+    logEdit(OP_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
   }
 
   /** 
    * Add generation stamp record to edit log
    */
   void logGenerationStamp(long genstamp) {
-    logEdit(Ops.OP_SET_GENSTAMP, new LongWritable(genstamp));
+    logEdit(OP_SET_GENSTAMP, new LongWritable(genstamp));
   }
 
   /** 
@@ -775,7 +742,7 @@ public class FSEditLog {
       new DeprecatedUTF8(src),
       FSEditLog.toLogLong(mtime),
       FSEditLog.toLogLong(atime)};
-    logEdit(Ops.OP_TIMES, new ArrayWritable(DeprecatedUTF8.class, info));
+    logEdit(OP_TIMES, new ArrayWritable(DeprecatedUTF8.class, info));
   }
 
   /** 
@@ -788,9 +755,9 @@ public class FSEditLog {
       new DeprecatedUTF8(value),
       FSEditLog.toLogLong(mtime),
       FSEditLog.toLogLong(atime)};
-    logEdit(Ops.OP_SYMLINK, 
-            new ArrayWritable(DeprecatedUTF8.class, info),
-            node.getPermissionStatus());
+    logEdit(OP_SYMLINK, 
+      new ArrayWritable(DeprecatedUTF8.class, info),
+      node.getPermissionStatus());
   }
   
   /**
@@ -801,20 +768,20 @@ public class FSEditLog {
    */
   void logGetDelegationToken(DelegationTokenIdentifier id,
       long expiryTime) {
-    logEdit(Ops.OP_GET_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
+    logEdit(OP_GET_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
   }
   
   void logRenewDelegationToken(DelegationTokenIdentifier id,
       long expiryTime) {
-    logEdit(Ops.OP_RENEW_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
+    logEdit(OP_RENEW_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
   }
   
   void logCancelDelegationToken(DelegationTokenIdentifier id) {
-    logEdit(Ops.OP_CANCEL_DELEGATION_TOKEN, id);
+    logEdit(OP_CANCEL_DELEGATION_TOKEN, id);
   }
   
   void logUpdateMasterKey(DelegationKey key) {
-    logEdit(Ops.OP_UPDATE_MASTER_KEY, key);
+    logEdit(OP_UPDATE_MASTER_KEY, key);
   }
   
   static private DeprecatedUTF8 toLogReplication(short replication) {
@@ -1063,7 +1030,7 @@ public class FSEditLog {
       boStream = new EditLogBackupOutputStream(bnReg, nnReg);
       editStreams.add(boStream);
     }
-    logEdit(Ops.OP_JSPOOL_START, (Writable[])null);
+    logEdit(OP_JSPOOL_START, (Writable[])null);
   }
 
   /**
@@ -1161,7 +1128,7 @@ public class FSEditLog {
   void incrementCheckpointTime() {
     fsimage.incrementCheckpointTime();
     Writable[] args = {new LongWritable(fsimage.getCheckpointTime())};
-    logEdit(Ops.OP_CHECKPOINT_TIME, args);
+    logEdit(OP_CHECKPOINT_TIME, args);
   }
 
   synchronized void releaseBackupStream(NamenodeRegistration registration) {

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Fri Jan 21 22:08:04 2011
@@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.security.t
 import static org.apache.hadoop.hdfs.server.common.Util.now;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLog.Ops;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Writable;
@@ -122,11 +121,12 @@ public class FSEditLogLoader {
         long mtime = 0;
         long atime = 0;
         long blockSize = 0;
-        byte opcode = -1;
+        FSEditLogOpCodes opCode;
         try {
           in.mark(1);
-          opcode = in.readByte();
-          if (opcode == Ops.OP_INVALID) {
+          byte opCodeByte = in.readByte();
+          opCode = FSEditLogOpCodes.fromByte(opCodeByte);
+          if (opCode == FSEditLogOpCodes.OP_INVALID) {
             in.reset(); // reset back to end of file if somebody reads it again
             break; // no more transactions
           }
@@ -134,9 +134,9 @@ public class FSEditLogLoader {
           break; // no more transactions
         }
         numEdits++;
-        switch (opcode) {
-        case Ops.OP_ADD:
-        case Ops.OP_CLOSE: {
+        switch (opCode) {
+        case OP_ADD:
+        case OP_CLOSE: {
           // versions > 0 support per file replication
           // get name and replication
           int length = in.readInt();
@@ -158,7 +158,7 @@ public class FSEditLogLoader {
             blockSize = readLong(in);
           }
           // get blocks
-          boolean isFileUnderConstruction = (opcode == Ops.OP_ADD);
+          boolean isFileUnderConstruction = (opCode == FSEditLogOpCodes.OP_ADD);
           BlockInfo blocks[] = 
             readBlocks(in, logVersion, isFileUnderConstruction, replication);
 
@@ -181,7 +181,7 @@ public class FSEditLogLoader {
           }
 
           // clientname, clientMachine and block locations of last block.
-          if (opcode == Ops.OP_ADD && logVersion <= -12) {
+          if (opCode == FSEditLogOpCodes.OP_ADD && logVersion <= -12) {
             clientName = FSImageSerialization.readString(in);
             clientMachine = FSImageSerialization.readString(in);
             if (-13 <= logVersion) {
@@ -195,7 +195,7 @@ public class FSEditLogLoader {
           // The open lease transaction re-creates a file if necessary.
           // Delete the file if it already exists.
           if (FSNamesystem.LOG.isDebugEnabled()) {
-            FSNamesystem.LOG.debug(opcode + ": " + path + 
+            FSNamesystem.LOG.debug(opCode + ": " + path + 
                                    " numblocks : " + blocks.length +
                                    " clientHolder " +  clientName +
                                    " clientMachine " + clientMachine);
@@ -229,16 +229,16 @@ public class FSEditLogLoader {
           }
           break;
         } 
-        case Ops.OP_SET_REPLICATION: {
+        case OP_SET_REPLICATION: {
           numOpSetRepl++;
           path = FSImageSerialization.readString(in);
           short replication = fsNamesys.adjustReplication(readShort(in));
           fsDir.unprotectedSetReplication(path, replication, null);
           break;
         } 
-        case Ops.OP_CONCAT_DELETE: {
+        case OP_CONCAT_DELETE: {
           if (logVersion > -22) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           numOpConcatDelete++;
@@ -257,7 +257,7 @@ public class FSEditLogLoader {
           fsDir.unprotectedConcat(trg, srcs);
           break;
         }
-        case Ops.OP_RENAME_OLD: {
+        case OP_RENAME_OLD: {
           numOpRenameOld++;
           int length = in.readInt();
           if (length != 3) {
@@ -272,7 +272,7 @@ public class FSEditLogLoader {
           fsNamesys.changeLease(s, d, dinfo);
           break;
         }
-        case Ops.OP_DELETE: {
+        case OP_DELETE: {
           numOpDelete++;
           int length = in.readInt();
           if (length != 2) {
@@ -284,7 +284,7 @@ public class FSEditLogLoader {
           fsDir.unprotectedDelete(path, timestamp);
           break;
         }
-        case Ops.OP_MKDIR: {
+        case OP_MKDIR: {
           numOpMkDir++;
           PermissionStatus permissions = fsNamesys.getUpgradePermission();
           int length = in.readInt();
@@ -309,47 +309,47 @@ public class FSEditLogLoader {
           fsDir.unprotectedMkdir(path, permissions, timestamp);
           break;
         }
-        case Ops.OP_SET_GENSTAMP: {
+        case OP_SET_GENSTAMP: {
           numOpSetGenStamp++;
           long lw = in.readLong();
           fsNamesys.setGenerationStamp(lw);
           break;
         } 
-        case Ops.OP_DATANODE_ADD: {
+        case OP_DATANODE_ADD: {
           numOpOther++;
           //Datanodes are not persistent any more.
           FSImageSerialization.DatanodeImage.skipOne(in);
           break;
         }
-        case Ops.OP_DATANODE_REMOVE: {
+        case OP_DATANODE_REMOVE: {
           numOpOther++;
           DatanodeID nodeID = new DatanodeID();
           nodeID.readFields(in);
           //Datanodes are not persistent any more.
           break;
         }
-        case Ops.OP_SET_PERMISSIONS: {
+        case OP_SET_PERMISSIONS: {
           numOpSetPerm++;
           if (logVersion > -11)
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                                   + " for version " + logVersion);
           fsDir.unprotectedSetPermission(
               FSImageSerialization.readString(in), FsPermission.read(in));
           break;
         }
-        case Ops.OP_SET_OWNER: {
+        case OP_SET_OWNER: {
           numOpSetOwner++;
           if (logVersion > -11)
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                                   + " for version " + logVersion);
           fsDir.unprotectedSetOwner(FSImageSerialization.readString(in),
               FSImageSerialization.readString_EmptyAsNull(in),
               FSImageSerialization.readString_EmptyAsNull(in));
           break;
         }
-        case Ops.OP_SET_NS_QUOTA: {
+        case OP_SET_NS_QUOTA: {
           if (logVersion > -16) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           fsDir.unprotectedSetQuota(FSImageSerialization.readString(in), 
@@ -357,9 +357,9 @@ public class FSEditLogLoader {
                                     FSConstants.QUOTA_DONT_SET);
           break;
         }
-        case Ops.OP_CLEAR_NS_QUOTA: {
+        case OP_CLEAR_NS_QUOTA: {
           if (logVersion > -16) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
@@ -368,14 +368,14 @@ public class FSEditLogLoader {
           break;
         }
 
-        case Ops.OP_SET_QUOTA:
+        case OP_SET_QUOTA:
           fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
                                     readLongWritable(in),
                                     readLongWritable(in));
                                       
           break;
 
-        case Ops.OP_TIMES: {
+        case OP_TIMES: {
           numOpTimes++;
           int length = in.readInt();
           if (length != 3) {
@@ -388,7 +388,7 @@ public class FSEditLogLoader {
           fsDir.unprotectedSetTimes(path, mtime, atime, true);
           break;
         }
-        case Ops.OP_SYMLINK: {
+        case OP_SYMLINK: {
           numOpSymlink++;
           int length = in.readInt();
           if (length != 4) {
@@ -403,9 +403,9 @@ public class FSEditLogLoader {
           fsDir.unprotectedSymlink(path, value, mtime, atime, perm);
           break;
         }
-        case Ops.OP_RENAME: {
+        case OP_RENAME: {
           if (logVersion > -21) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           numOpRename++;
@@ -423,9 +423,9 @@ public class FSEditLogLoader {
           fsNamesys.changeLease(s, d, dinfo);
           break;
         }
-        case Ops.OP_GET_DELEGATION_TOKEN: {
+        case OP_GET_DELEGATION_TOKEN: {
           if (logVersion > -24) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           numOpGetDelegationToken++;
@@ -437,9 +437,9 @@ public class FSEditLogLoader {
               .addPersistedDelegationToken(delegationTokenId, expiryTime);
           break;
         }
-        case Ops.OP_RENEW_DELEGATION_TOKEN: {
+        case OP_RENEW_DELEGATION_TOKEN: {
           if (logVersion > -24) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           numOpRenewDelegationToken++;
@@ -451,9 +451,9 @@ public class FSEditLogLoader {
               .updatePersistedTokenRenewal(delegationTokenId, expiryTime);
           break;
         }
-        case Ops.OP_CANCEL_DELEGATION_TOKEN: {
+        case OP_CANCEL_DELEGATION_TOKEN: {
           if (logVersion > -24) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           numOpCancelDelegationToken++;
@@ -464,9 +464,9 @@ public class FSEditLogLoader {
               .updatePersistedTokenCancellation(delegationTokenId);
           break;
         }
-        case Ops.OP_UPDATE_MASTER_KEY: {
+        case OP_UPDATE_MASTER_KEY: {
           if (logVersion > -24) {
-            throw new IOException("Unexpected opcode " + opcode
+            throw new IOException("Unexpected opCode " + opCode
                 + " for version " + logVersion);
           }
           numOpUpdateMasterKey++;
@@ -477,7 +477,7 @@ public class FSEditLogLoader {
           break;
         }
         default: {
-          throw new IOException("Never seen opcode " + opcode);
+          throw new IOException("Never seen opCode " + opCode);
         }
         }
       }

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Op codes for edits file
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum FSEditLogOpCodes {
+  // last op code in file
+  OP_INVALID                    ((byte) -1),
+  OP_ADD                        ((byte)  0),
+  OP_RENAME_OLD                 ((byte)  1), // deprecated operation
+  OP_DELETE                     ((byte)  2),
+  OP_MKDIR                      ((byte)  3),
+  OP_SET_REPLICATION            ((byte)  4),
+  @Deprecated OP_DATANODE_ADD   ((byte)  5),
+  @Deprecated OP_DATANODE_REMOVE((byte)  6),
+  OP_SET_PERMISSIONS            ((byte)  7),
+  OP_SET_OWNER                  ((byte)  8),
+  OP_CLOSE                      ((byte)  9),
+  OP_SET_GENSTAMP               ((byte) 10),
+  OP_SET_NS_QUOTA               ((byte) 11), // obsolete
+  OP_CLEAR_NS_QUOTA             ((byte) 12), // obsolete
+  OP_TIMES                      ((byte) 13), // set atime, mtime
+  OP_SET_QUOTA                  ((byte) 14),
+  OP_RENAME                     ((byte) 15), // filecontext rename
+  OP_CONCAT_DELETE              ((byte) 16), // concat files
+  OP_SYMLINK                    ((byte) 17),
+  OP_GET_DELEGATION_TOKEN       ((byte) 18),
+  OP_RENEW_DELEGATION_TOKEN     ((byte) 19),
+  OP_CANCEL_DELEGATION_TOKEN    ((byte) 20),
+  OP_UPDATE_MASTER_KEY          ((byte) 21),
+  // must be same as NamenodeProtocol.JA_JSPOOL_START
+  OP_JSPOOL_START               ((byte)102),
+  // must be same as NamenodeProtocol.JA_CHECKPOINT_TIME
+  OP_CHECKPOINT_TIME            ((byte)103);
+
+  private byte opCode;
+
+  /**
+   * Constructor
+   *
+   * @param opCode byte value of constructed enum
+   */
+  FSEditLogOpCodes(byte opCode) {
+    this.opCode = opCode;
+  }
+
+  /**
+   * return the byte value of the enum
+   *
+   * @return the byte value of the enum
+   */
+  public byte getOpCode() {
+    return opCode;
+  }
+
+  private static final Map<Byte, FSEditLogOpCodes> byteToEnum =
+    new HashMap<Byte, FSEditLogOpCodes>();
+
+  static {
+    // initialize byte to enum map
+    for(FSEditLogOpCodes opCode : values())
+      byteToEnum.put(opCode.getOpCode(), opCode);
+  }
+
+  /**
+   * Converts byte to FSEditLogOpCodes enum value
+   *
+   * @param opCode get enum for this opCode
+   * @return enum with byte value of opCode
+   */
+  public static FSEditLogOpCodes fromByte(byte opCode) {
+    return byteToEnum.get(opCode);
+  }
+}

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java Fri Jan 21 22:08:04 2011
@@ -54,8 +54,8 @@ public interface NamenodeProtocol extend
   // Journal action codes. See journal().
   public static byte JA_IS_ALIVE = 100; // check whether the journal is alive
   public static byte JA_JOURNAL      = 101; // just journal
-  public static byte JA_JSPOOL_START = 102;  // = FSEditLog.OP_JSPOOL_START
-  public static byte JA_CHECKPOINT_TIME = 103; // = FSEditLog.OP_CHECKPOINT_TIME
+  public static byte JA_JSPOOL_START = 102;  // = FSEditLogOpCodes.OP_JSPOOL_START
+  public static byte JA_CHECKPOINT_TIME = 103; // = FSEditLogOpCodes.OP_CHECKPOINT_TIME
 
   public final static int ACT_UNKNOWN = 0;    // unknown action   
   public final static int ACT_SHUTDOWN = 50;   // shutdown node

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.FileOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * BinaryEditsVisitor implements a binary EditsVisitor
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class BinaryEditsVisitor extends EditsVisitor {
+  final private DataOutputStream out;
+
+  /**
+   * Create a processor that writes to a given file and
+   * reads using a given Tokenizer
+   *
+   * @param filename Name of file to write output to
+   * @param tokenizer Input tokenizer
+   */
+  public BinaryEditsVisitor(String filename, Tokenizer tokenizer)
+    throws IOException {
+
+    this(filename, tokenizer, false);
+  }
+
+  /**
+   * Create a processor that writes to a given file and reads using
+   * a given Tokenizer, may also print to screen
+   *
+   * @param filename Name of file to write output to
+   * @param tokenizer Input tokenizer
+   * @param printToScreen Mirror output to screen? (ignored for binary)
+   */
+  public BinaryEditsVisitor(String filename,
+    Tokenizer tokenizer,
+    boolean printToScreen) throws IOException {
+
+    super(tokenizer);
+    out = new DataOutputStream(new FileOutputStream(filename));
+  }
+
+  /**
+   * Start the visitor (initialization)
+   */
+  @Override
+  void start() throws IOException {
+    // nothing to do for binary format
+  }
+
+  /**
+   * Finish the visitor
+   */
+  @Override
+  void finish() throws IOException {
+    close();
+  }
+
+  /**
+   * Finish the visitor and indicate an error
+   */
+  @Override
+  void finishAbnormally() throws IOException {
+    System.err.println("Error processing EditLog file.  Exiting.");
+    close();
+  }
+
+  /**
+   * Close output stream and prevent further writing
+   */
+  private void close() throws IOException {
+    out.close();
+  }
+
+  /**
+   * Visit a enclosing element (element that has other elements in it)
+   */
+  @Override
+  void visitEnclosingElement(Tokenizer.Token value) throws IOException {
+    // nothing to do for binary format
+  }
+
+  /**
+   * End of eclosing element
+   */
+  @Override
+  void leaveEnclosingElement() throws IOException {
+    // nothing to do for binary format
+  }  
+
+  /**
+   * Visit a Token
+   */
+  @Override
+  Tokenizer.Token visit(Tokenizer.Token value) throws IOException {
+    value.toBinary(out);
+    return value;
+  }
+}

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.FileInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Tokenizer that reads tokens from a binary file
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class BinaryTokenizer implements Tokenizer {
+
+  private DataInputStream in;
+
+  /**
+   * BinaryTokenizer constructor
+   *
+   * @param filename input filename
+   */
+  public BinaryTokenizer(String filename) throws FileNotFoundException {
+    in = new DataInputStream(new FileInputStream(filename));
+  }
+
+  /**
+   * BinaryTokenizer constructor
+   *
+   * @param in input stream
+   */
+  public BinaryTokenizer(DataInputStream in) throws IOException {
+    this.in = in;
+  }
+
+  /**
+   * @see org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer#read
+   *
+   * @param t a Token to read
+   * @return token that was just read
+   */
+  @Override
+  public Token read(Token t) throws IOException {
+    t.fromBinary(in);
+    return t;
+  }
+}

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Structural elements of an EditLog that may be encountered within the
+ * file. EditsVisitor is able to process these elements.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum EditsElement {
+  EDITS,
+  EDITS_VERSION,
+  RECORD,
+  OPCODE,
+  DATA,
+    // elements in the data part of the editLog records
+    LENGTH,
+    // for OP_SET_GENSTAMP
+    GENERATION_STAMP,
+    // for OP_ADD, OP_CLOSE
+    PATH,
+    REPLICATION,
+    MTIME,
+    ATIME,
+    BLOCKSIZE,
+    NUMBLOCKS,
+    BLOCK,
+      BLOCK_ID,
+      BLOCK_NUM_BYTES,
+      BLOCK_GENERATION_STAMP,
+    PERMISSION_STATUS,
+      FS_PERMISSIONS,
+    CLIENT_NAME,
+    CLIENT_MACHINE,
+    // for OP_RENAME_OLD
+    SOURCE,
+    DESTINATION,
+    TIMESTAMP,
+    // for OP_SET_OWNER
+    USERNAME,
+    GROUPNAME,
+    // for OP_SET_QUOTA
+    NS_QUOTA,
+    DS_QUOTA,
+    // for OP_RENAME
+    RENAME_OPTIONS,
+    // for OP_CONCAT_DELETE
+    CONCAT_TARGET,
+    CONCAT_SOURCE,
+    // for OP_GET_DELEGATION_TOKEN
+    T_VERSION,
+    T_OWNER,
+    T_RENEWER,
+    T_REAL_USER,
+    T_ISSUE_DATE,
+    T_MAX_DATE,
+    T_SEQUENCE_NUMBER,
+    T_MASTER_KEY_ID,
+    T_EXPIRY_TIME,
+    // for OP_UPDATE_MASTER_KEY
+    KEY_ID,
+    KEY_EXPIRY_DATE,
+    KEY_LENGTH,
+    KEY_BLOB
+}

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An EditsLoader can read a Hadoop EditLog file  and walk over its
+ * structure using the supplied EditsVisitor.
+ *
+ * Each implementation of EditsLoader is designed to rapidly process an
+ * edits log file.  As long as minor changes are made from one layout version
+ * to another, it is acceptable to tweak one implementation to read the next.
+ * However, if the layout version changes enough that it would make a
+ * processor slow or difficult to read, another processor should be created.
+ * This allows each processor to quickly read an edits log without getting
+ * bogged down in dealing with significant differences between layout versions.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface EditsLoader {
+
+  /**
+   * Loads the edits file
+   */
+  public void loadEdits() throws IOException;
+
+  /**
+   * Can this processor handle the specified version of EditLog file?
+   *
+   * @param version EditLog version file
+   * @return True if this instance can process the file
+   */
+  public boolean canLoadVersion(int version);
+
+  /**
+   * Factory for obtaining version of edits log loader that can read
+   * a particular edits log format.
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  public class LoaderFactory {
+    // Java doesn't support static methods on interfaces, which necessitates
+    // this factory class
+
+    /**
+     * Create an edits log loader, at this point we only have one,
+     * we might need to add more later
+     *
+     * @param v an instance of EditsVisitor (binary, XML etc.)
+     * @return EditsLoader that can interpret specified version, or null
+     */
+    static public EditsLoader getLoader(EditsVisitor v) {
+      return new EditsLoaderCurrent(v);
+    }
+  }
+}

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.EOFException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
+
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ByteToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ShortToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.IntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VIntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.LongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VLongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringUTF8Token;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringTextToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BlobToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BytesWritableToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.EmptyToken;
+
+/**
+ * EditsLoaderCurrent processes Hadoop EditLogs files and walks over
+ * them using a provided EditsVisitor, calling the visitor at each element
+ * enumerated below.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class EditsLoaderCurrent implements EditsLoader {
+
+  private static int [] supportedVersions = {
+    -18, -19, -20, -21, -22, -23, -24, -25, -26, -27 };
+
+  private EditsVisitor v;
+  private int editsVersion = 0;
+
+  /**
+   * Constructor
+   */
+  public EditsLoaderCurrent(EditsVisitor visitor) {
+    v = visitor;
+  }
+
+  /**
+   * Checks if this EditsLoader can load given version of edits
+   *
+   * @param version version to load
+   * @return true if this EditsLoader can load given version of edits
+   */
+  @Override
+  public boolean canLoadVersion(int version) {
+    for(int v : supportedVersions) { if(v == version) return true; }
+    return false;
+  }
+
+  /**
+   * Visit OP_INVALID
+   */
+  private void visit_OP_INVALID() throws IOException {
+    ; // nothing to do, this op code has no data
+  }
+
+  /**
+   * Visit OP_ADD
+   */
+  private void visit_OP_ADD() throws IOException {
+    visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes.OP_ADD);
+  }
+
+  /**
+   * Visit OP_CLOSE
+   */
+  private void visit_OP_CLOSE() throws IOException {
+    visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes.OP_CLOSE);
+  }
+
+  /**
+   * Visit OP_ADD and OP_CLOSE, they are almost the same
+   *
+   * @param editsOpCode op code to visit
+   */
+  private void visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes editsOpCode)
+    throws IOException {
+
+    IntToken opAddLength = v.visitInt(EditsElement.LENGTH);
+    // this happens if the edits is not properly ended (-1 op code),
+    // it is padded at the end with all zeros, OP_ADD is zero so
+    // without this check we would treat all zeros as empty OP_ADD)
+    if(opAddLength.value == 0) {
+      throw new IOException("OpCode " + editsOpCode +
+        " has zero length (corrupted edits)");
+    }
+    v.visitStringUTF8(EditsElement.PATH);
+    v.visitStringUTF8(EditsElement.REPLICATION);
+    v.visitStringUTF8(EditsElement.MTIME);
+    v.visitStringUTF8(EditsElement.ATIME);
+    v.visitStringUTF8(EditsElement.BLOCKSIZE);
+    // now read blocks
+    IntToken numBlocksToken = v.visitInt(EditsElement.NUMBLOCKS);
+    for (int i = 0; i < numBlocksToken.value; i++) {
+      v.visitEnclosingElement(EditsElement.BLOCK);
+
+      v.visitLong(EditsElement.BLOCK_ID);
+      v.visitLong(EditsElement.BLOCK_NUM_BYTES);
+      v.visitLong(EditsElement.BLOCK_GENERATION_STAMP);
+
+      v.leaveEnclosingElement();
+    }
+    // PERMISSION_STATUS
+    v.visitEnclosingElement(EditsElement.PERMISSION_STATUS);
+
+    v.visitStringText( EditsElement.USERNAME);
+    v.visitStringText( EditsElement.GROUPNAME);
+    v.visitShort(      EditsElement.FS_PERMISSIONS);
+
+    v.leaveEnclosingElement();
+    if(editsOpCode == FSEditLogOpCodes.OP_ADD) {
+      v.visitStringUTF8(EditsElement.CLIENT_NAME);
+      v.visitStringUTF8(EditsElement.CLIENT_MACHINE);
+    }
+  }
+
+  /**
+   * Visit OP_RENAME_OLD
+   */
+  private void visit_OP_RENAME_OLD() throws IOException {
+    v.visitInt(        EditsElement.LENGTH);
+    v.visitStringUTF8( EditsElement.SOURCE);
+    v.visitStringUTF8( EditsElement.DESTINATION);
+    v.visitStringUTF8( EditsElement.TIMESTAMP);
+  }
+
+  /**
+   * Visit OP_DELETE
+   */
+  private void visit_OP_DELETE() throws IOException {
+    v.visitInt(        EditsElement.LENGTH);
+    v.visitStringUTF8( EditsElement.PATH);
+    v.visitStringUTF8( EditsElement.TIMESTAMP);
+  }
+
+  /**
+   * Visit OP_MKDIR
+   */
+  private void visit_OP_MKDIR() throws IOException {
+    v.visitInt(        EditsElement.LENGTH);
+    v.visitStringUTF8( EditsElement.PATH);
+    v.visitStringUTF8( EditsElement.TIMESTAMP);
+    v.visitStringUTF8( EditsElement.ATIME);
+    // PERMISSION_STATUS
+    v.visitEnclosingElement( EditsElement.PERMISSION_STATUS);
+
+    v.visitStringText( EditsElement.USERNAME);
+    v.visitStringText( EditsElement.GROUPNAME);
+    v.visitShort(      EditsElement.FS_PERMISSIONS);
+
+    v.leaveEnclosingElement();
+  }
+
+  /**
+   * Visit OP_SET_REPLICATION
+   */
+  private void visit_OP_SET_REPLICATION() throws IOException {
+    v.visitStringUTF8(EditsElement.PATH);
+    v.visitStringUTF8(EditsElement.REPLICATION);
+  }
+
+  /**
+   * Visit OP_SET_PERMISSIONS
+   */
+  private void visit_OP_SET_PERMISSIONS() throws IOException {
+    v.visitStringUTF8( EditsElement.PATH);
+    v.visitShort(      EditsElement.FS_PERMISSIONS);
+  }
+
+  /**
+   * Visit OP_SET_OWNER
+   */
+  private void visit_OP_SET_OWNER() throws IOException {
+    v.visitStringUTF8(EditsElement.PATH);
+    v.visitStringUTF8(EditsElement.USERNAME);
+    v.visitStringUTF8(EditsElement.GROUPNAME);
+  }
+
+  /**
+   * Visit OP_SET_GENSTAMP
+   */
+  private void visit_OP_SET_GENSTAMP() throws IOException {
+    v.visitLong(EditsElement.GENERATION_STAMP);
+  }
+
+  /**
+   * Visit OP_TIMES
+   */
+  private void visit_OP_TIMES() throws IOException {
+    v.visitInt(        EditsElement.LENGTH);
+    v.visitStringUTF8( EditsElement.PATH);
+    v.visitStringUTF8( EditsElement.MTIME);
+    v.visitStringUTF8( EditsElement.ATIME);
+  }
+
+  /**
+   * Visit OP_SET_QUOTA
+   */
+  private void visit_OP_SET_QUOTA() throws IOException {
+    v.visitStringUTF8( EditsElement.PATH);
+    v.visitLong(       EditsElement.NS_QUOTA);
+    v.visitLong(       EditsElement.DS_QUOTA);
+  }
+
+  /**
+   * Visit OP_RENAME
+   */
+  private void visit_OP_RENAME() throws IOException {
+    if(editsVersion > -21) {
+      throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_RENAME
+        + " for edit log version " + editsVersion
+        + " (op code 15 only expected for 21 and later)");
+    }
+    v.visitInt(           EditsElement.LENGTH);
+    v.visitStringUTF8(    EditsElement.SOURCE);
+    v.visitStringUTF8(    EditsElement.DESTINATION);
+    v.visitStringUTF8(    EditsElement.TIMESTAMP);
+    v.visitBytesWritable( EditsElement.RENAME_OPTIONS);
+  }
+
+  /**
+   * Visit OP_CONCAT_DELETE
+   */
+  private void visit_OP_CONCAT_DELETE() throws IOException {
+    if(editsVersion > -22) {
+      throw new IOException("Unexpected op code "
+        + FSEditLogOpCodes.OP_CONCAT_DELETE
+        + " for edit log version " + editsVersion
+        + " (op code 16 only expected for 22 and later)");
+    }
+    IntToken lengthToken = v.visitInt(EditsElement.LENGTH);
+    v.visitStringUTF8(EditsElement.CONCAT_TARGET);
+    // all except of CONCAT_TARGET and TIMESTAMP
+    int sourceCount = lengthToken.value - 2;
+    for(int i = 0; i < sourceCount; i++) {
+      v.visitStringUTF8(EditsElement.CONCAT_SOURCE);
+    }
+    v.visitStringUTF8(EditsElement.TIMESTAMP);
+  }
+
+  /**
+   * Visit OP_SYMLINK
+   */
+  private void visit_OP_SYMLINK() throws IOException {
+    v.visitInt(        EditsElement.LENGTH);
+    v.visitStringUTF8( EditsElement.SOURCE);
+    v.visitStringUTF8( EditsElement.DESTINATION);
+    v.visitStringUTF8( EditsElement.MTIME);
+    v.visitStringUTF8( EditsElement.ATIME);
+    // PERMISSION_STATUS
+    v.visitEnclosingElement(EditsElement.PERMISSION_STATUS);
+
+    v.visitStringText( EditsElement.USERNAME);
+    v.visitStringText( EditsElement.GROUPNAME);
+    v.visitShort(      EditsElement.FS_PERMISSIONS);
+
+    v.leaveEnclosingElement();
+  }
+
+  /**
+   * Visit OP_GET_DELEGATION_TOKEN
+   */
+  private void visit_OP_GET_DELEGATION_TOKEN() throws IOException {
+      if(editsVersion > -24) {
+        throw new IOException("Unexpected op code "
+          + FSEditLogOpCodes.OP_GET_DELEGATION_TOKEN
+          + " for edit log version " + editsVersion
+          + " (op code 18 only expected for 24 and later)");
+      }
+      v.visitByte(       EditsElement.T_VERSION);
+      v.visitStringText( EditsElement.T_OWNER);
+      v.visitStringText( EditsElement.T_RENEWER);
+      v.visitStringText( EditsElement.T_REAL_USER);
+      v.visitVLong(      EditsElement.T_ISSUE_DATE);
+      v.visitVLong(      EditsElement.T_MAX_DATE);
+      v.visitVInt(       EditsElement.T_SEQUENCE_NUMBER);
+      v.visitVInt(       EditsElement.T_MASTER_KEY_ID);
+      v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
+  }
+
+  /**
+   * Visit OP_RENEW_DELEGATION_TOKEN
+   */
+  private void visit_OP_RENEW_DELEGATION_TOKEN()
+    throws IOException {
+
+      if(editsVersion > -24) {
+        throw new IOException("Unexpected op code "
+          + FSEditLogOpCodes.OP_RENEW_DELEGATION_TOKEN
+          + " for edit log version " + editsVersion
+          + " (op code 19 only expected for 24 and later)");
+      }
+      v.visitByte(       EditsElement.T_VERSION);
+      v.visitStringText( EditsElement.T_OWNER);
+      v.visitStringText( EditsElement.T_RENEWER);
+      v.visitStringText( EditsElement.T_REAL_USER);
+      v.visitVLong(      EditsElement.T_ISSUE_DATE);
+      v.visitVLong(      EditsElement.T_MAX_DATE);
+      v.visitVInt(       EditsElement.T_SEQUENCE_NUMBER);
+      v.visitVInt(       EditsElement.T_MASTER_KEY_ID);
+      v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
+  }
+
+  /**
+   * Visit OP_CANCEL_DELEGATION_TOKEN
+   */
+  private void visit_OP_CANCEL_DELEGATION_TOKEN()
+    throws IOException {
+
+      if(editsVersion > -24) {
+        throw new IOException("Unexpected op code "
+          + FSEditLogOpCodes.OP_CANCEL_DELEGATION_TOKEN
+          + " for edit log version " + editsVersion
+          + " (op code 20 only expected for 24 and later)");
+      }
+      v.visitByte(       EditsElement.T_VERSION);
+      v.visitStringText( EditsElement.T_OWNER);
+      v.visitStringText( EditsElement.T_RENEWER);
+      v.visitStringText( EditsElement.T_REAL_USER);
+      v.visitVLong(      EditsElement.T_ISSUE_DATE);
+      v.visitVLong(      EditsElement.T_MAX_DATE);
+      v.visitVInt(       EditsElement.T_SEQUENCE_NUMBER);
+      v.visitVInt(       EditsElement.T_MASTER_KEY_ID);
+  }
+
+  /**
+   * Visit OP_UPDATE_MASTER_KEY
+   */
+  private void visit_OP_UPDATE_MASTER_KEY()
+    throws IOException {
+
+      if(editsVersion > -24) {
+        throw new IOException("Unexpected op code "
+          + FSEditLogOpCodes.OP_UPDATE_MASTER_KEY
+          + " for edit log version " + editsVersion
+          + "(op code 21 only expected for 24 and later)");
+      }
+      v.visitVInt(  EditsElement.KEY_ID);
+      v.visitVLong( EditsElement.KEY_EXPIRY_DATE);
+      VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH);
+      v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value);
+  }
+
+  private void visitOpCode(FSEditLogOpCodes editsOpCode)
+    throws IOException {
+
+    switch(editsOpCode) {
+      case OP_INVALID: // -1
+        visit_OP_INVALID();
+        break;
+      case OP_ADD: // 0
+        visit_OP_ADD();
+        break;
+      case OP_CLOSE: // 9
+        visit_OP_CLOSE();
+        break;
+      case OP_RENAME_OLD: // 1
+        visit_OP_RENAME_OLD();
+        break;
+      case OP_DELETE: // 2
+        visit_OP_DELETE();
+        break;
+      case OP_MKDIR: // 3
+        visit_OP_MKDIR();
+        break;
+      case OP_SET_REPLICATION: // 4
+        visit_OP_SET_REPLICATION();
+        break;
+      case OP_SET_PERMISSIONS: // 7
+        visit_OP_SET_PERMISSIONS();
+        break;
+      case OP_SET_OWNER: // 8
+        visit_OP_SET_OWNER();
+        break;
+      case OP_SET_GENSTAMP: // 10
+        visit_OP_SET_GENSTAMP();
+        break;
+      case OP_TIMES: // 13
+        visit_OP_TIMES();
+        break;
+      case OP_SET_QUOTA: // 14
+        visit_OP_SET_QUOTA();
+        break;
+      case OP_RENAME: // 15
+        visit_OP_RENAME();
+        break;
+      case OP_CONCAT_DELETE: // 16
+        visit_OP_CONCAT_DELETE();
+        break;
+      case OP_SYMLINK: // 17
+        visit_OP_SYMLINK();
+        break;
+      case OP_GET_DELEGATION_TOKEN: // 18
+        visit_OP_GET_DELEGATION_TOKEN();
+        break;
+      case OP_RENEW_DELEGATION_TOKEN: // 19
+        visit_OP_RENEW_DELEGATION_TOKEN();
+        break;
+      case OP_CANCEL_DELEGATION_TOKEN: // 20
+        visit_OP_CANCEL_DELEGATION_TOKEN();
+        break;
+      case OP_UPDATE_MASTER_KEY: // 21
+        visit_OP_UPDATE_MASTER_KEY();
+        break;
+      default:
+      {
+        throw new IOException("Unknown op code " + editsOpCode);
+      }
+    }
+  }
+
+  /**
+   * Loads edits file, uses visitor to process all elements
+   */
+  @Override
+  public void loadEdits() throws IOException {
+
+    try {
+      v.start();
+      v.visitEnclosingElement(EditsElement.EDITS);
+
+      IntToken editsVersionToken = v.visitInt(EditsElement.EDITS_VERSION);
+      editsVersion = editsVersionToken.value;
+      if(!canLoadVersion(editsVersion)) {
+        throw new IOException("Cannot process editLog version " +
+          editsVersionToken.value);
+      }
+
+      FSEditLogOpCodes editsOpCode;
+      do {
+        v.visitEnclosingElement(EditsElement.RECORD);
+
+        ByteToken opCodeToken = v.visitByte(EditsElement.OPCODE);
+        editsOpCode = FSEditLogOpCodes.fromByte(opCodeToken.value);
+
+        v.visitEnclosingElement(EditsElement.DATA);
+
+        visitOpCode(editsOpCode);
+
+        v.leaveEnclosingElement(); // DATA
+        v.leaveEnclosingElement(); // RECORD
+      } while(editsOpCode != FSEditLogOpCodes.OP_INVALID);
+
+      v.leaveEnclosingElement(); // EDITS
+      v.finish();
+    } catch(IOException e) {
+      // Tell the visitor to clean up, then re-throw the exception
+      v.finishAbnormally();
+      throw e;
+    }
+  }
+}

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.hadoop.hdfs.DeprecatedUTF8;
+
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ByteToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ShortToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.IntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VIntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.LongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VLongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringUTF8Token;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringTextToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BlobToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BytesWritableToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.EmptyToken;
+
+/**
+ * An implementation of EditsVisitor can traverse the structure of an
+ * Hadoop edits log and respond to each of the structures within the file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+abstract public class EditsVisitor {
+
+  private Tokenizer tokenizer;
+
+  public EditsVisitor(Tokenizer tokenizer) {
+    this.tokenizer = tokenizer;
+  }
+
+  /**
+   * Begin visiting the edits log structure.  Opportunity to perform
+   * any initialization necessary for the implementing visitor.
+   */
+  abstract void start() throws IOException;
+
+  /**
+   * Finish visiting the edits log structure.  Opportunity to perform any
+   * clean up necessary for the implementing visitor.
+   */
+  abstract void finish() throws IOException;
+
+  /**
+   * Finish visiting the edits log structure after an error has occurred
+   * during the processing.  Opportunity to perform any clean up necessary
+   * for the implementing visitor.
+   */
+  abstract void finishAbnormally() throws IOException;
+
+  /**
+   * Visit non enclosing element of edits log with specified value.
+   *
+   * @param value a token to visit
+   */
+  abstract Tokenizer.Token visit(Tokenizer.Token value) throws IOException;
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public ByteToken visitByte(EditsElement e) throws IOException {
+    return (ByteToken)visit(tokenizer.read(new ByteToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public ShortToken visitShort(EditsElement e) throws IOException {
+    return (ShortToken)visit(tokenizer.read(new ShortToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public IntToken visitInt(EditsElement e) throws IOException {
+    return (IntToken)visit(tokenizer.read(new IntToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public VIntToken visitVInt(EditsElement e) throws IOException {
+    return (VIntToken)visit(tokenizer.read(new VIntToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public LongToken visitLong(EditsElement e) throws IOException {
+    return (LongToken)visit(tokenizer.read(new LongToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public VLongToken visitVLong(EditsElement e) throws IOException {
+    return (VLongToken)visit(tokenizer.read(new VLongToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public StringUTF8Token visitStringUTF8(EditsElement e) throws IOException {
+    return (StringUTF8Token)visit(tokenizer.read(new StringUTF8Token(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public StringTextToken visitStringText(EditsElement e) throws IOException {
+    return (StringTextToken)visit(tokenizer.read(new StringTextToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public BlobToken visitBlob(EditsElement e, int length) throws IOException {
+    return (BlobToken)visit(tokenizer.read(new BlobToken(e, length)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public BytesWritableToken visitBytesWritable(EditsElement e) throws IOException {
+    return (BytesWritableToken)visit(tokenizer.read(new BytesWritableToken(e)));
+  }
+
+  /**
+   * Convenience shortcut method to parse a specific token type
+   */
+  public EmptyToken visitEmpty(EditsElement e) throws IOException {
+    return (EmptyToken)visit(tokenizer.read(new EmptyToken(e)));
+  }
+
+  /**
+   * Begin visiting an element that encloses another element, such as
+   * the beginning of the list of blocks that comprise a file.
+   *
+   * @param value Token being visited
+   */
+  abstract void visitEnclosingElement(Tokenizer.Token value)
+     throws IOException;
+
+  /**
+   * Convenience shortcut method (it virutally always uses EmptyToken)
+   */
+  void visitEnclosingElement(EditsElement e) throws IOException {
+    visitEnclosingElement(tokenizer.read(new EmptyToken(e)));
+  }
+
+  /**
+   * Leave current enclosing element.  Called, for instance, at the end of
+   * processing the blocks that compromise a file.
+   */
+  abstract void leaveEnclosingElement() throws IOException;
+}

Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * EditsVisitorFactory for different implementations of EditsVisitor
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class EditsVisitorFactory {
+
+  /**
+   * Factory function that creates an EditsVisitor object
+   *
+   * @param filename output filename
+   * @param tokenizer input tokenizer
+   * @return EditsVisitor for appropriate output format (binary, XML etc.)
+   */
+  static public EditsVisitor getEditsVisitor(String filename,
+    String processor,
+    Tokenizer tokenizer,
+    boolean printToScreen) throws IOException {
+
+    if(processor.toLowerCase().equals("xml")) {
+      return new XmlEditsVisitor(filename, tokenizer, printToScreen);
+    } else if(processor.toLowerCase().equals("stats")) {
+      return new StatisticsEditsVisitor(filename, tokenizer, printToScreen);
+    } else if(processor.toLowerCase().equals("binary")) {
+      return new BinaryEditsVisitor(filename, tokenizer, printToScreen);
+    } else {
+      throw new IOException("Unknown proccesor " + processor +
+        " (valid processors: xml, binary, stats)");
+    }
+  }
+}



Mime
View raw message