hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1098785 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/ src/test/hdfs/org/apache/...
Date Mon, 02 May 2011 20:44:14 GMT
Author: suresh
Date: Mon May  2 20:44:13 2011
New Revision: 1098785

URL: http://svn.apache.org/viewvc?rev=1098785&view=rev
Log:
Merging change r1090419 for HDFS-1813 from federation

Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/build.xml   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,5 +1,5 @@
 /hadoop/core/branches/branch-0.19/hdfs:713112
-/hadoop/hdfs/branches/HDFS-1052:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
 /hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1036738,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1098785&r1=1098784&r2=1098785&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Mon May  2 20:44:13 2011
@@ -229,6 +229,9 @@ Trunk (unreleased changes)
     HDFS-1791. Federation: Add command to delete block pool directories 
     from a datanode. (jitendra)
 
+    HDFS-1813. Federation: Authentication using BlockToken in RPC to datanode 
+               fails. (jitendra)
+
   IMPROVEMENTS
 
     HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)

Propchange: hadoop/hdfs/branches/yahoo-merge/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
 /hadoop/core/trunk/build.xml:779102
-/hadoop/hdfs/branches/HDFS-1052/build.xml:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/build.xml:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
 /hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
 /hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
 /hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java:713112
 /hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/java:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
 /hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1098785&r1=1098784&r2=1098785&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Mon May  2 20:44:13 2011
@@ -1367,6 +1367,9 @@ public class DataNode extends Configured
     registerMXBean();
     initDataXceiver(conf);
     startInfoServer(conf);
+  
+    // BlockPoolTokenSecretManager is required to create ipc server.
+    this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();
     initIpcServer(conf);
 
     metrics = DataNodeMetrics.create(conf, datanodeId.getName());
@@ -2006,10 +2009,6 @@ public class DataNode extends Configured
     dataXceiverServer.start();
     ipcServer.start();
     startPlugins(conf);
-    
-    // BlockPoolTokenSecretManager is created here, but it shouldn't be
-    // used until it is initialized in register().
-    this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();
   }
 
   /**
@@ -2526,7 +2525,7 @@ public class DataNode extends Configured
           LOG.debug("Got: " + id.toString());
         }
         blockPoolTokenSecretManager.checkAccess(id, null, block,
-            BlockTokenSecretManager.AccessMode.WRITE);
+            BlockTokenSecretManager.AccessMode.READ);
       }
     }
 

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -2,7 +2,7 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:713112
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
 /hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1098785&r1=1098784&r2=1098785&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Mon May  2 20:44:13 2011
@@ -905,18 +905,17 @@ public class FSNamesystem implements FSC
       if (LOG.isDebugEnabled()) {
         LOG.debug("last = " + last);
       }
-
-      if(isBlockTokenEnabled && needBlockToken) {
+      
+      LocatedBlock lastBlock = last.isComplete() ? blockManager
+          .getBlockLocation(last, n - last.getNumBytes()) : blockManager
+          .getBlockLocation(last, n);
+          
+      if (isBlockTokenEnabled && needBlockToken) {
         setBlockTokens(locatedblocks);
+        setBlockToken(lastBlock);
       }
-
-      if (last.isComplete()) {
-        return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
-          blockManager.getBlockLocation(last, n-last.getNumBytes()), true);
-      } else {
-        return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
-          blockManager.getBlockLocation(last, n), false);
-      }
+      return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
+          lastBlock, last.isComplete());
     }
     } finally {
       readUnlock();
@@ -932,13 +931,16 @@ public class FSNamesystem implements FSC
   /** Generate block tokens for the blocks to be returned. */
   private void setBlockTokens(List<LocatedBlock> locatedBlocks) throws IOException
{
     for(LocatedBlock l : locatedBlocks) {
-      Token<BlockTokenIdentifier> token = 
-        blockTokenSecretManager.generateToken(l.getBlock(), 
-            EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
-    
-      l.setBlockToken(token);
+      setBlockToken(l);
     }
   }
+  
+  /** Generate block token for a LocatedBlock. */
+  private void setBlockToken(LocatedBlock l) throws IOException {
+    Token<BlockTokenIdentifier> token = blockTokenSecretManager.generateToken(l
+        .getBlock(), EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
+    l.setBlockToken(token);
+  }
 
   /**
    * Moves all the blocks from srcs and appends them to trg

Propchange: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
 /hadoop/core/trunk/src/test/hdfs:776175-785643
-/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
 /hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Modified: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1098785&r1=1098784&r2=1098785&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Mon May  2 20:44:13 2011
@@ -29,9 +29,16 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.io.TestWritable;
 import org.apache.hadoop.ipc.Client;
 import org.apache.hadoop.ipc.RPC;
@@ -45,6 +52,7 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.log4j.Level;
 
+import org.junit.Assert;
 import org.junit.Test;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
@@ -251,4 +259,43 @@ public class TestBlockToken {
       tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid));
     }
   }
+  
+  /**
+   * This test writes a file and gets the block locations without closing
+   * the file, and tests the block token in the last block. Block token is
+   * verified by ensuring it is of correct kind.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testBlockTokenInLastLocatedBlock() throws IOException,
+      InterruptedException {
+    Configuration conf = new HdfsConfiguration();
+    conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numNameNodes(1)
+        .numDataNodes(1).build();
+    cluster.waitActive();
+
+    try {
+      FileSystem fs = cluster.getFileSystem();
+      String fileName = "/testBlockTokenInLastLocatedBlock";
+      Path filePath = new Path(fileName);
+      FSDataOutputStream out = fs.create(filePath, (short) 1);
+      out.write(new byte[1000]);
+      LocatedBlocks locatedBlocks = cluster.getNameNode().getBlockLocations(
+          fileName, 0, 1000);
+      while (locatedBlocks.getLastLocatedBlock() == null) {
+        Thread.sleep(100);
+        locatedBlocks = cluster.getNameNode().getBlockLocations(fileName, 0,
+            1000);
+      }
+      Token<BlockTokenIdentifier> token = locatedBlocks.getLastLocatedBlock()
+          .getBlockToken();
+      Assert.assertEquals(BlockTokenIdentifier.KIND_NAME, token.getKind());
+      out.close();
+    } finally {
+      cluster.shutdown();
+    }
+  } 
 }

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
 /hadoop/core/trunk/src/webapps/datanode:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
 /hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
 /hadoop/core/trunk/src/webapps/hdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
 /hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May  2 20:44:13 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
 /hadoop/core/trunk/src/webapps/secondary:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
 /hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1052823,1060619,1061067,1062020,1062045,1062052



Mime
View raw message