hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1462877 - in /hadoop/common/branches/branch-1.2: ./ bin/ src/core/org/apache/hadoop/io/compress/ src/docs/src/documentation/content/xdocs/ src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/hadoo...
Date Sun, 31 Mar 2013 04:34:02 GMT
Author: suresh
Date: Sun Mar 31 04:34:01 2013
New Revision: 1462877

URL: http://svn.apache.org/r1462877
Log:
HDFS-4651. Merge changes 1462875 and 1462876 from branch-1

Added:
    hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/hdfs_imageviewer.xml
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/hdfs_imageviewer.xml
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/
      - copied from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/protocol/
      - copied from r1462876, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/protocol/
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
      - copied unchanged from r1462876, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/
      - copied from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/
      - copied from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
    hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
      - copied unchanged from r1462875, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
Modified:
    hadoop/common/branches/branch-1.2/bin/hadoop
    hadoop/common/branches/branch-1.2/build.xml
    hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
    hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

Modified: hadoop/common/branches/branch-1.2/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/bin/hadoop?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/bin/hadoop (original)
+++ hadoop/common/branches/branch-1.2/bin/hadoop Sun Mar 31 04:34:01 2013
@@ -81,6 +81,7 @@ print_usage()
   echo "  fsck                 run a DFS filesystem checking utility"
   echo "  fs                   run a generic filesystem user client"
   echo "  balancer             run a cluster balancing utility"
+  echo "  oiv                  apply the offline fsimage viewer to an fsimage"
   echo "  fetchdt              fetch a delegation token from the NameNode"
   echo "  jobtracker           run the MapReduce job Tracker node" 
   echo "  pipes                run a Pipes job"
@@ -287,6 +288,8 @@ elif [ "$COMMAND" = "fsck" ] ; then
 elif [ "$COMMAND" = "balancer" ] ; then
   CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS"
+elif [ "$COMMAND" = "oiv" ] ; then
+  CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
 elif [ "$COMMAND" = "fetchdt" ] ; then
   CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
 elif [ "$COMMAND" = "jobtracker" ] ; then

Modified: hadoop/common/branches/branch-1.2/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/build.xml?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/build.xml (original)
+++ hadoop/common/branches/branch-1.2/build.xml Sun Mar 31 04:34:01 2013
@@ -936,6 +936,8 @@
     <copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data120bytes" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18"
todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19"
todir="${test.cache.data}"/>
   </target>
 
   <!-- ================================================================== -->

Modified: hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
(original)
+++ hadoop/common/branches/branch-1.2/src/core/org/apache/hadoop/io/compress/CompressionCodecFactory.java
Sun Mar 31 04:34:01 2013
@@ -39,10 +39,30 @@ public class CompressionCodecFactory {
    * automatically supports finding the longest matching suffix. 
    */
   private SortedMap<String, CompressionCodec> codecs = null;
+
+    /**
+     * A map from the reversed filename suffixes to the codecs.
+     * This is probably overkill, because the maps should be small, but it
+     * automatically supports finding the longest matching suffix.
+     */
+    private Map<String, CompressionCodec> codecsByName = null;
+
+  /**
+   * A map from class names to the codecs
+   */
+  private HashMap<String, CompressionCodec> codecsByClassName = null;
   
   private void addCodec(CompressionCodec codec) {
     String suffix = codec.getDefaultExtension();
     codecs.put(new StringBuffer(suffix).reverse().toString(), codec);
+    codecsByClassName.put(codec.getClass().getCanonicalName(), codec);
+
+    String codecName = codec.getClass().getSimpleName();
+    codecsByName.put(codecName.toLowerCase(), codec);
+    if (codecName.endsWith("Codec")) {
+      codecName = codecName.substring(0, codecName.length() - "Codec".length());
+      codecsByName.put(codecName.toLowerCase(), codec);
+    }
   }
   
   /**
@@ -131,6 +151,8 @@ public class CompressionCodecFactory {
    */
   public CompressionCodecFactory(Configuration conf) {
     codecs = new TreeMap<String, CompressionCodec>();
+    codecsByClassName = new HashMap<String, CompressionCodec>();
+    codecsByName = new HashMap<String, CompressionCodec>();
     List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf);
     if (codecClasses == null) {
       addCodec(new GzipCodec());
@@ -168,6 +190,68 @@ public class CompressionCodecFactory {
   }
   
   /**
+   * Find the relevant compression codec for the codec's canonical class name.
+   * @param classname the canonical class name of the codec
+   * @return the codec object
+   */
+  public CompressionCodec getCodecByClassName(String classname) {
+    if (codecsByClassName == null) {
+      return null;
+    }
+    return codecsByClassName.get(classname);
+  }
+
+    /**
+     * Find the relevant compression codec for the codec's canonical class name
+     * or by codec alias.
+     * <p/>
+     * Codec aliases are case insensitive.
+     * <p/>
+     * The code alias is the short class name (without the package name).
+     * If the short class name ends with 'Codec', then there are two aliases for
+     * the codec, the complete short class name and the short class name without
+     * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+     * alias are 'gzip' and 'gzipcodec'.
+     *
+     * @param codecName the canonical class name of the codec
+     * @return the codec object
+     */
+    public CompressionCodec getCodecByName(String codecName) {
+      if (codecsByClassName == null) {
+        return null;
+      }
+      CompressionCodec codec = getCodecByClassName(codecName);
+      if (codec == null) {
+        // trying to get the codec by name in case the name was specified instead a class
+        codec = codecsByName.get(codecName.toLowerCase());
+      }
+      return codec;
+    }
+
+    /**
+     * Find the relevant compression codec for the codec's canonical class name
+     * or by codec alias and returns its implemetation class.
+     * <p/>
+     * Codec aliases are case insensitive.
+     * <p/>
+     * The code alias is the short class name (without the package name).
+     * If the short class name ends with 'Codec', then there are two aliases for
+     * the codec, the complete short class name and the short class name without
+     * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+     * alias are 'gzip' and 'gzipcodec'.
+     *
+     * @param codecName the canonical class name of the codec
+     * @return the codec class
+     */
+    public Class<? extends CompressionCodec> getCodecClassByName(String codecName)
{
+      CompressionCodec codec = getCodecByName(codecName);
+      if (codec == null) {
+        return null;
+      }
+      return codec.getClass();
+    }
+  
+  /**
    * Removes a suffix from a filename, if it has it.
    * @param filename the filename to strip
    * @param suffix the suffix to remove

Modified: hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/common/branches/branch-1.2/src/docs/src/documentation/content/xdocs/site.xml Sun
Mar 31 04:34:01 2013
@@ -63,6 +63,7 @@ See http://forrest.apache.org/docs/linki
     <hdfs_perm        label="Permissions" href="hdfs_permissions_guide.html" />
     <hdfs_quotas      label="Quotas" href="hdfs_quota_admin_guide.html" />
     <hdfs_SLG         label="Synthetic Load Generator"  href="SLG_user_guide.html" />
+    <hdfs_imageviewer	label="Offline Image Viewer"	href="hdfs_imageviewer.html" />
     <hftp label="HFTP" href="hftp.html"/>
     <webhdfs label="WebHDFS REST API" href="webhdfs.html" />
     <hdfs_libhdfs       label="C API libhdfs" href="libhdfs.html" />

Modified: hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Sun
Mar 31 04:34:01 2013
@@ -130,6 +130,8 @@ public class DFSConfigKeys extends Commo
   public static final long    DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24*60*60*1000;
   public static final String  DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY = "dfs.namenode.delegation.token.max-lifetime";
   public static final long    DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7*24*60*60*1000;
+  public static final String  DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY = "dfs.namenode.delegation.token.always-use";
// for tests
+  public static final boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
 
   //Following keys have no defaults
   public static final String  DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir";

Modified: hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Sun Mar 31 04:34:01 2013
@@ -1820,7 +1820,7 @@ public class FSImage extends Storage {
   }
 
   static private final UTF8 U_STR = new UTF8();
-  static String readString(DataInputStream in) throws IOException {
+  public static String readString(DataInputStream in) throws IOException {
     U_STR.readFields(in);
     return U_STR.toString();
   }
@@ -1830,7 +1830,7 @@ public class FSImage extends Storage {
     return s.isEmpty()? null: s;
   }
 
-  static byte[] readBytes(DataInputStream in) throws IOException {
+  public static byte[] readBytes(DataInputStream in) throws IOException {
     U_STR.readFields(in);
     int len = U_STR.getLength();
     byte[] bytes = new byte[len];

Modified: hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1462877&r1=1462876&r2=1462877&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/common/branches/branch-1.2/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Sun Mar 31 04:34:01 2013
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT;
+
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -294,8 +297,15 @@ public class NameNode implements ClientP
     
     myMetrics = NameNodeInstrumentation.create(conf);
     this.namesystem = new FSNamesystem(this, conf);
+    
+    // For testing purposes, allow the DT secret manager to be started regardless
+    // of whether security is enabled.
+    boolean alwaysUseDelegationTokensForTests = 
+      conf.getBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
+          DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
 
-    if (UserGroupInformation.isSecurityEnabled()) {
+    if (UserGroupInformation.isSecurityEnabled() ||
+        alwaysUseDelegationTokensForTests) {
       namesystem.activateSecretManager();
     }
 



Mime
View raw message