hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077404 - in /hadoop/common/branches/branch-0.20-security-patches/src: hdfs/org/apache/hadoop/hdfs/protocol/ hdfs/org/apache/hadoop/hdfs/security/token/block/ hdfs/org/apache/hadoop/hdfs/server/namenode/ hdfs/org/apache/hadoop/hdfs/server/...
Date Fri, 04 Mar 2011 04:11:34 GMT
Author: omalley
Date: Fri Mar  4 04:11:33 2011
New Revision: 1077404

URL: http://svn.apache.org/viewvc?rev=1077404&view=rev
Log:
commit 4da5e8870ddcf1b2f57777aec05df90f8b193543
Author: Jakob Homan <jhoman@yahoo-inc.com>
Date:   Fri Apr 16 17:35:14 2010 -0700

    HDFS:1081 from https://issues.apache.org/jira/secure/attachment/12442023/HADOOP-1081-Y20-2.patch
    
    +++ b/YAHOO-CHANGES.txt
    +    HDFS-1081. Performance regression in
    +    DistributedFileSystem::getFileBlockLocations in secure systems (jhoman)
    +

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
    hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
    hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
    hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1077404&r1=1077403&r2=1077404&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
Fri Mar  4 04:11:33 2011
@@ -50,9 +50,12 @@ public interface ClientProtocol extends 
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 45: Replace full getListing with iterative getListinng
+   * 61: Serialized format of BlockTokenIdentifier changed to contain
+   *     multiple blocks within a single BlockTokenIdentifier 
+   *     
+   *     (bumped to 61 to bring in line with trunk)
    */
-  public static final long versionID = 45L;
+  public static final long versionID = 61L;
   
   ///////////////////////////////////////
   // File contents

Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java?rev=1077404&r1=1077403&r2=1077404&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
Fri Mar  4 04:11:33 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.security.
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.EnumSet;
 
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
@@ -35,17 +36,23 @@ public class BlockTokenIdentifier extend
   private long expiryDate;
   private int keyId;
   private String userId;
-  private long blockId;
+  private long [] blockIds;
   private EnumSet<AccessMode> modes;
 
+  private byte [] cache;
+  
   public BlockTokenIdentifier() {
-    this(null, 0, EnumSet.noneOf(AccessMode.class));
+    this(null, new long [] {}, EnumSet.noneOf(AccessMode.class));
   }
 
-  public BlockTokenIdentifier(String userId, long blockId,
+  public BlockTokenIdentifier(String userId, long [] blockIds,
       EnumSet<AccessMode> modes) {
+    if(blockIds == null) 
+      throw new IllegalArgumentException("blockIds can't be null");
+    this.cache = null;
     this.userId = userId;
-    this.blockId = blockId;
+    this.blockIds = Arrays.copyOf(blockIds, blockIds.length);
+    Arrays.sort(this.blockIds);
     this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
   }
 
@@ -57,7 +64,7 @@ public class BlockTokenIdentifier extend
   @Override
   public UserGroupInformation getUser() {
     if (userId == null || "".equals(userId)) {
-      return UserGroupInformation.createRemoteUser(Long.toString(blockId));
+      return UserGroupInformation.createRemoteUser(Arrays.toString(blockIds));
     }
     return UserGroupInformation.createRemoteUser(userId);
   }
@@ -67,6 +74,7 @@ public class BlockTokenIdentifier extend
   }
 
   public void setExpiryDate(long expiryDate) {
+    cache = null;
     this.expiryDate = expiryDate;
   }
 
@@ -75,6 +83,7 @@ public class BlockTokenIdentifier extend
   }
 
   public void setKeyId(int keyId) {
+    cache = null;
     this.keyId = keyId;
   }
 
@@ -82,18 +91,36 @@ public class BlockTokenIdentifier extend
     return userId;
   }
 
-  public long getBlockId() {
-    return blockId;
+  /**
+   * Return sorted array of blockIds this {@link BlockTokenIdentifier} includes
+   */
+  public long [] getBlockIds() {
+    return blockIds;
+  }
+  
+  /**
+   * Is specified blockId included in this BlockTokenIdentifier?
+   */
+  public boolean isBlockIncluded(long blockId) {
+    switch(blockIds.length) {
+    case 1:
+      return blockIds[0] == blockId;
+    case 2:
+      return (blockIds[0] == blockId) || (blockIds[1] == blockId);
+    default:
+      return Arrays.binarySearch(blockIds, blockId) >= 0;  
+    }
   }
 
   public EnumSet<AccessMode> getAccessModes() {
     return modes;
   }
 
+  @Override
   public String toString() {
     return "block_token_identifier (expiryDate=" + this.getExpiryDate()
         + ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId()
-        + ", blockId=" + this.getBlockId() + ", access modes="
+        + ", blockIds=" + Arrays.toString(blockIds) + ", access modes="
         + this.getAccessModes() + ")";
   }
 
@@ -109,7 +136,8 @@ public class BlockTokenIdentifier extend
     if (obj instanceof BlockTokenIdentifier) {
       BlockTokenIdentifier that = (BlockTokenIdentifier) obj;
       return this.expiryDate == that.expiryDate && this.keyId == that.keyId
-          && isEqual(this.userId, that.userId) && this.blockId == that.blockId
+          && isEqual(this.userId, that.userId) 
+          && Arrays.equals(this.blockIds, that.blockIds)
           && isEqual(this.modes, that.modes);
     }
     return false;
@@ -117,15 +145,18 @@ public class BlockTokenIdentifier extend
 
   /** {@inheritDoc} */
   public int hashCode() {
-    return (int) expiryDate ^ keyId ^ (int) blockId ^ modes.hashCode()
+    return (int) expiryDate ^ keyId ^ Arrays.hashCode(blockIds) ^ modes.hashCode()
         ^ (userId == null ? 0 : userId.hashCode());
   }
 
   public void readFields(DataInput in) throws IOException {
+    cache = null;
     expiryDate = WritableUtils.readVLong(in);
     keyId = WritableUtils.readVInt(in);
     userId = WritableUtils.readString(in);
-    blockId = WritableUtils.readVLong(in);
+    blockIds = new long[WritableUtils.readVInt(in)];
+    for(int i = 0; i < blockIds.length; i++)
+      blockIds[i] = WritableUtils.readVLong(in);
     int length = WritableUtils.readVInt(in);
     for (int i = 0; i < length; i++) {
       modes.add(WritableUtils.readEnum(in, AccessMode.class));
@@ -136,10 +167,19 @@ public class BlockTokenIdentifier extend
     WritableUtils.writeVLong(out, expiryDate);
     WritableUtils.writeVInt(out, keyId);
     WritableUtils.writeString(out, userId);
-    WritableUtils.writeVLong(out, blockId);
+    WritableUtils.writeVInt(out, blockIds.length);
+    for(int i = 0; i < blockIds.length; i++)
+      WritableUtils.writeVLong(out, blockIds[i]);
     WritableUtils.writeVInt(out, modes.size());
     for (AccessMode aMode : modes) {
       WritableUtils.writeEnum(out, aMode);
     }
   }
+  
+  @Override
+  public byte[] getBytes() {
+    if(cache == null) cache = super.getBytes();
+    
+    return cache;
+  }
 }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java?rev=1077404&r1=1077403&r2=1077404&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
Fri Mar  4 04:11:33 2011
@@ -174,16 +174,29 @@ public class BlockTokenSecretManager ext
   /** Generate an block token for current user */
   public Token<BlockTokenIdentifier> generateToken(Block block,
       EnumSet<AccessMode> modes) throws IOException {
-    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    String userID = (ugi == null ? null : ugi.getShortUserName());
-    return generateToken(userID, block, modes);
+    return generateToken(new long [] { block.getBlockId() }, modes);
   }
 
   /** Generate a block token for a specified user */
   public Token<BlockTokenIdentifier> generateToken(String userId, Block block,
       EnumSet<AccessMode> modes) throws IOException {
-    BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
-        .getBlockId(), modes);
+    return generateToken(userId, new long [] { block.getBlockId() }, modes);
+  }
+
+  /** Generate a block token for the current user based on a collection
+   * of blockIds
+   */
+  public Token<BlockTokenIdentifier> generateToken(long[] blockIds,
+      EnumSet<AccessMode> modes) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String userID = (ugi == null ? null : ugi.getShortUserName());
+    return generateToken(userID, blockIds, modes);
+  }
+  
+  /** Generate a block token based on a collection of blockIds */
+  public Token<BlockTokenIdentifier> generateToken(String userID,
+      long[] blockIds, EnumSet<AccessMode> modes) {
+    BlockTokenIdentifier id = new BlockTokenIdentifier(userID, blockIds, modes);
     return new Token<BlockTokenIdentifier>(id, this);
   }
 
@@ -202,7 +215,7 @@ public class BlockTokenSecretManager ext
       throw new InvalidToken("Block token with " + id.toString()
           + " doesn't belong to user " + userId);
     }
-    if (id.getBlockId() != block.getBlockId()) {
+    if (!id.isBlockIncluded(block.getBlockId())) {
       throw new InvalidToken("Block token with " + id.toString()
           + " doesn't apply to block " + block);
     }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1077404&r1=1077403&r2=1077404&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Fri Mar  4 04:11:33 2011
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.*;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@@ -919,10 +920,7 @@ public class FSNamesystem implements FSC
       }
       LocatedBlock b = new LocatedBlock(blocks[curBlk], machineSet, curPos,
           blockCorrupt);
-      if (isAccessTokenEnabled) {
-        b.setBlockToken(accessTokenHandler.generateToken(b.getBlock(), 
-            EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
-      }
+
       results.add(b); 
       curPos += blocks[curBlk].getNumBytes();
       curBlk++;
@@ -930,6 +928,24 @@ public class FSNamesystem implements FSC
           && curBlk < blocks.length 
           && results.size() < nrBlocksToReturn);
     
+    if(isAccessTokenEnabled) {
+      // Generate a list of the blockIds to be returned for this request
+      long [] blockIds = new long[results.size()];
+      for(int i = 0; i < results.size(); i++) {
+        blockIds[i] = results.get(i).getBlock().getBlockId();
+      }
+      
+      // Generate a single BlockTokenIdentifier for all ids
+      Token<BlockTokenIdentifier> bti 
+        = accessTokenHandler.generateToken(blockIds, 
+            EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
+      
+      // Assign a reference to this BlockTokenIdentifier to all blocks
+      for(LocatedBlock lb : results) {
+        lb.setBlockToken(bti);
+      }
+    }
+    
     return inode.createLocatedBlocks(results);
   }
 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=1077404&r1=1077403&r2=1077404&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
Fri Mar  4 04:11:33 2011
@@ -41,10 +41,12 @@ import org.apache.hadoop.security.Kerber
     clientPrincipal = DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY)
 public interface DatanodeProtocol extends VersionedProtocol {
   /**
-   * 20: SendHeartbeat may return KeyUpdateCommand
-   *     Register returns access keys inside DatanodeRegistration object
+   * 25: Serialized format of BlockTokenIdentifier changed to contain
+   *     multiple blocks within a single BlockTokenIdentifier
+   *     
+   *     (bumped to 25 to bring in line with trunk)
    */
-  public static final long versionID = 20L;
+  public static final long versionID = 25L;
   
   // error code
   final static int NOTIFY = 0;

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1077404&r1=1077403&r2=1077404&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Fri Mar  4 04:11:33 2011
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Set;
 
@@ -111,7 +112,7 @@ public class TestBlockToken {
         LOG.info("Got: " + id.toString());
         assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
         sm.checkAccess(id, null, block, BlockTokenSecretManager.AccessMode.WRITE);
-        result = new LocatedBlock(new Block(id.getBlockId()), null);
+        result = new LocatedBlock(new Block(id.getBlockIds()[0]), null);
       }
       return result;
     }
@@ -225,4 +226,28 @@ public class TestBlockToken {
     }
   }
 
+  @Test
+  public void collectionOfBlocksActsSanely() {
+    final long[][] testBlockIds = new long [][] {{99l, 7l, -32l, 0l},
+                                                 {},
+                                                 {42l},
+                                                 {-5235l, 2352}};
+    final long [] notBlockIds = new long [] { 32l, 1l, -23423423l};
+    
+    for(long [] bids : testBlockIds) {
+      BlockTokenIdentifier bti = new BlockTokenIdentifier("Madame Butterfly", 
+          bids, EnumSet.noneOf(BlockTokenSecretManager.AccessMode.class));
+      
+      for(long bid : bids) assertTrue(bti.isBlockIncluded(bid));
+      
+      for(long nbid : notBlockIds) assertFalse(bti.isBlockIncluded(nbid));
+      
+      // BlockTokenIdentifiers maintain a sorted array of the block Ids.
+      long[] sorted = Arrays.copyOf(bids, bids.length);
+      Arrays.sort(sorted);
+      
+      assertTrue(Arrays.toString(bids)+" doesn't equal "+Arrays.toString(sorted), 
+          Arrays.equals(bti.getBlockIds(), sorted));
+    }
+  }
 }



Mime
View raw message