hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject svn commit: r992515 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/security/token/block/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/server/protocol/ src/test/h...
Date Sat, 04 Sep 2010 00:41:31 GMT
Author: jghoman
Date: Sat Sep  4 00:41:30 2010
New Revision: 992515

URL: http://svn.apache.org/viewvc?rev=992515&view=rev
Log:
HDFS-1353.  Remove most of getBlockLocation optimization.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Sat Sep  4 00:41:30 2010
@@ -234,6 +234,8 @@ Trunk (unreleased changes)
     HDFS-1355. ant veryclean (clean-cache) doesn't clean enough. 
     (Luke Lu via jghoman)
 
+    HDFS-1353.  Remove most of getBlockLocation optimization. (jghoman)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Sat Sep
 4 00:41:30 2010
@@ -68,9 +68,9 @@ public interface ClientProtocol extends 
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 62: Allow iterative getListinng piggyback block locations.
+   * 63: remove getBlockLocations optimization
    */
-  public static final long versionID = 62L;
+  public static final long versionID = 63L;
   
   ///////////////////////////////////////
   // File contents

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
Sat Sep  4 00:41:30 2010
@@ -21,7 +21,6 @@ package org.apache.hadoop.hdfs.security.
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.EnumSet;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -38,23 +37,20 @@ public class BlockTokenIdentifier extend
   private long expiryDate;
   private int keyId;
   private String userId;
-  private long [] blockIds;
+  private long blockId;
   private EnumSet<AccessMode> modes;
 
   private byte [] cache;
   
   public BlockTokenIdentifier() {
-    this(null, new long [] {}, EnumSet.noneOf(AccessMode.class));
+    this(null, 0, EnumSet.noneOf(AccessMode.class));
   }
 
-  public BlockTokenIdentifier(String userId, long [] blockIds,
+  public BlockTokenIdentifier(String userId, long blockId,
       EnumSet<AccessMode> modes) {
-    if(blockIds == null) 
-      throw new IllegalArgumentException("blockIds can't be null");
     this.cache = null;
     this.userId = userId;
-    this.blockIds = Arrays.copyOf(blockIds, blockIds.length);
-    Arrays.sort(this.blockIds);
+    this.blockId = blockId;
     this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
   }
 
@@ -66,7 +62,7 @@ public class BlockTokenIdentifier extend
   @Override
   public UserGroupInformation getUser() {
     if (userId == null || "".equals(userId)) {
-      return UserGroupInformation.createRemoteUser(Arrays.toString(blockIds));
+      return UserGroupInformation.createRemoteUser(Long.toString(blockId));
     }
     return UserGroupInformation.createRemoteUser(userId);
   }
@@ -93,27 +89,10 @@ public class BlockTokenIdentifier extend
     return userId;
   }
 
-  /**
-   * Return sorted array of blockIds this {@link BlockTokenIdentifier} includes
-   */
-  public long [] getBlockIds() {
-    return blockIds;
+  public long getBlockId() {
+    return blockId;
   }
-  
-  /**
-   * Is specified blockId included in this BlockTokenIdentifier?
-   */
-  public boolean isBlockIncluded(long blockId) {
-    switch(blockIds.length) {
-    case 1:
-      return blockIds[0] == blockId;
-    case 2:
-      return (blockIds[0] == blockId) || (blockIds[1] == blockId);
-    default:
-      return Arrays.binarySearch(blockIds, blockId) >= 0;  
-    }
-  }
-  
+
   public EnumSet<AccessMode> getAccessModes() {
     return modes;
   }
@@ -122,7 +101,7 @@ public class BlockTokenIdentifier extend
   public String toString() {
     return "block_token_identifier (expiryDate=" + this.getExpiryDate()
         + ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId()
-        + ", blockId=" + Arrays.toString(blockIds) + ", access modes="
+        + ", blockId=" + this.getBlockId() + ", access modes="
         + this.getAccessModes() + ")";
   }
 
@@ -138,8 +117,7 @@ public class BlockTokenIdentifier extend
     if (obj instanceof BlockTokenIdentifier) {
       BlockTokenIdentifier that = (BlockTokenIdentifier) obj;
       return this.expiryDate == that.expiryDate && this.keyId == that.keyId
-          && isEqual(this.userId, that.userId) 
-          && Arrays.equals(this.blockIds, that.blockIds)
+          && isEqual(this.userId, that.userId) && this.blockId == that.blockId
           && isEqual(this.modes, that.modes);
     }
     return false;
@@ -147,18 +125,16 @@ public class BlockTokenIdentifier extend
 
   /** {@inheritDoc} */
   public int hashCode() {
-    return (int) expiryDate ^ keyId ^ Arrays.hashCode(blockIds) ^ modes.hashCode()
+    return (int) expiryDate ^ keyId ^ (int) blockId ^ modes.hashCode()
         ^ (userId == null ? 0 : userId.hashCode());
   }
 
   public void readFields(DataInput in) throws IOException {
-    cache = null;
+    this.cache = null;
     expiryDate = WritableUtils.readVLong(in);
     keyId = WritableUtils.readVInt(in);
     userId = WritableUtils.readString(in);
-    blockIds = new long[WritableUtils.readVInt(in)];
-    for(int i = 0; i < blockIds.length; i++)
-      blockIds[i] = WritableUtils.readVLong(in);
+    blockId = WritableUtils.readVLong(in);
     int length = WritableUtils.readVInt(in);
     for (int i = 0; i < length; i++) {
       modes.add(WritableUtils.readEnum(in, AccessMode.class));
@@ -169,9 +145,7 @@ public class BlockTokenIdentifier extend
     WritableUtils.writeVLong(out, expiryDate);
     WritableUtils.writeVInt(out, keyId);
     WritableUtils.writeString(out, userId);
-    WritableUtils.writeVInt(out, blockIds.length);
-    for(int i = 0; i < blockIds.length; i++)
-      WritableUtils.writeVLong(out, blockIds[i]);
+    WritableUtils.writeVLong(out, blockId);
     WritableUtils.writeVInt(out, modes.size());
     for (AccessMode aMode : modes) {
       WritableUtils.writeEnum(out, aMode);

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
Sat Sep  4 00:41:30 2010
@@ -176,29 +176,16 @@ public class BlockTokenSecretManager ext
   /** Generate an block token for current user */
   public Token<BlockTokenIdentifier> generateToken(Block block,
       EnumSet<AccessMode> modes) throws IOException {
-    return generateToken(new long [] { block.getBlockId() }, modes);
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String userID = (ugi == null ? null : ugi.getShortUserName());
+    return generateToken(userID, block, modes);
   }
 
   /** Generate a block token for a specified user */
   public Token<BlockTokenIdentifier> generateToken(String userId, Block block,
       EnumSet<AccessMode> modes) throws IOException {
-    return generateToken(userId, new long [] { block.getBlockId() }, modes);
-  }
-
-  /** Generate a block token for the current user based on a collection
-   * of blockIds
-   */
-  public Token<BlockTokenIdentifier> generateToken(long[] blockIds,
-      EnumSet<AccessMode> modes) throws IOException {
-    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    String userID = (ugi == null ? null : ugi.getShortUserName());
-    return generateToken(userID, blockIds, modes);
-  }
-  
-  /** Generate a block token based on a collection of blockIds */
-  public Token<BlockTokenIdentifier> generateToken(String userID,
-      long[] blockIds, EnumSet<AccessMode> modes) {
-    BlockTokenIdentifier id = new BlockTokenIdentifier(userID, blockIds, modes);
+    BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
+        .getBlockId(), modes);
     return new Token<BlockTokenIdentifier>(id, this);
   }
 
@@ -217,7 +204,7 @@ public class BlockTokenSecretManager ext
       throw new InvalidToken("Block token with " + id.toString()
           + " doesn't belong to user " + userId);
     }
-    if (!id.isBlockIncluded(block.getBlockId())) {
+    if (id.getBlockId() != block.getBlockId()) {
       throw new InvalidToken("Block token with " + id.toString()
           + " doesn't apply to block " + block);
     }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Sat
Sep  4 00:41:30 2010
@@ -384,7 +384,7 @@ public class BlockManager {
   }
 
   List<LocatedBlock> getBlockLocations(BlockInfo[] blocks, long offset,
-      long length, int nrBlocksToReturn) throws IOException {
+      long length, int nrBlocksToReturn, boolean needBlockToken) throws IOException {
     int curBlk = 0;
     long curPos = 0, blkSize = 0;
     int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length;
@@ -403,7 +403,7 @@ public class BlockManager {
     long endOff = offset + length;
     List<LocatedBlock> results = new ArrayList<LocatedBlock>(blocks.length);
     do {
-      results.add(getBlockLocation(blocks[curBlk], curPos));
+      results.add(getBlockLocation(blocks[curBlk], curPos, needBlockToken));
       curPos += blocks[curBlk].getNumBytes();
       curBlk++;
     } while (curPos < endOff 
@@ -412,13 +412,14 @@ public class BlockManager {
     return results;
   }
 
-  /** @return a LocatedBlock for the given block */
-  LocatedBlock getBlockLocation(final BlockInfo blk, final long pos
+  /** @param needBlockToken 
+   * @return a LocatedBlock for the given block */
+  LocatedBlock getBlockLocation(final BlockInfo blk, final long pos, boolean needBlockToken
       ) throws IOException {
     if (!blk.isComplete()) {
       final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk;
       final DatanodeDescriptor[] locations = uc.getExpectedLocations();
-      return namesystem.createLocatedBlock(uc, locations, pos, false);
+      return namesystem.createLocatedBlock(uc, locations, pos, false, needBlockToken);
     }
 
     // get block locations
@@ -444,7 +445,7 @@ public class BlockManager {
           machines[j++] = d;
       }
     }
-    return namesystem.createLocatedBlock(blk, machines, pos, isCorrupt);    
+    return namesystem.createLocatedBlock(blk, machines, pos, isCorrupt, needBlockToken);
   
   }
 
   /**

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat
Sep  4 00:41:30 2010
@@ -24,7 +24,6 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.conf.*;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.*;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@@ -799,44 +798,32 @@ public class FSNamesystem implements FSC
     } else {
       final long n = inode.computeFileSize(false);
       final List<LocatedBlock> locatedblocks = blockManager.getBlockLocations(
-          blocks, offset, length, Integer.MAX_VALUE);
+          blocks, offset, length, Integer.MAX_VALUE, needBlockToken);
       final BlockInfo last = inode.getLastBlock();
       if (LOG.isDebugEnabled()) {
         LOG.debug("last = " + last);
       }
-
-      if(isBlockTokenEnabled && needBlockToken) {
-        setBlockTokens(locatedblocks);
-      }
-
+    
       if (last.isComplete()) {
         return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
-            blockManager.getBlockLocation(last, n-last.getNumBytes()), true);
+          blockManager.getBlockLocation(last, n-last.getNumBytes(), needBlockToken), true);
       } else {
         return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
-            blockManager.getBlockLocation(last, n), false);
+          blockManager.getBlockLocation(last, n, needBlockToken), false);
       }
     }
   }
 
-  /** Generate a combined block token for all the blocks to be returned. */
-  private void setBlockTokens(List<LocatedBlock> locatedBlocks) throws IOException
{
-    long [] blockIds = new long[locatedBlocks.size()];
-    for(int i = 0; i < blockIds.length; i++) {
-      blockIds[i] = locatedBlocks.get(i).getBlock().getBlockId();
-    }
-    
-    Token<BlockTokenIdentifier> token = 
-      blockTokenSecretManager.generateToken(blockIds, 
-          EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
-    
-    for(LocatedBlock l : locatedBlocks) l.setBlockToken(token);
-  }
-
-  /** Create a LocatedBlock. */
+  /** Create a LocatedBlock. 
+   * @param needBlockToken */
   LocatedBlock createLocatedBlock(final Block b, final DatanodeInfo[] locations,
-      final long offset, final boolean corrupt) throws IOException {
-    return new LocatedBlock(b, locations, offset, corrupt);
+      final long offset, final boolean corrupt, boolean needBlockToken) throws IOException
{
+    final LocatedBlock lb = new LocatedBlock(b, locations, offset, corrupt);
+    if (isBlockTokenEnabled && needBlockToken) {
+      lb.setBlockToken(blockTokenSecretManager.generateToken(b,
+          EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
+    }
+    return lb;
   }
   
   /**

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
Sat Sep  4 00:41:30 2010
@@ -44,9 +44,9 @@ import org.apache.avro.reflect.Nullable;
 @InterfaceAudience.Private
 public interface DatanodeProtocol extends VersionedProtocol {
   /**
-   * 25: HDFS-1081. Performance optimization on getBlocksLocation()
+   * 26: remove getBlockLocations optimization
    */
-  public static final long versionID = 25L;
+  public static final long versionID = 26;
   
   // error code
   final static int NOTIFY = 0;

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Sat Sep  4 00:41:30 2010
@@ -22,7 +22,6 @@ import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Set;
 
@@ -103,16 +102,15 @@ public class TestBlockToken {
       Set<TokenIdentifier> tokenIds = UserGroupInformation.getCurrentUser()
           .getTokenIdentifiers();
       assertEquals("Only one BlockTokenIdentifier expected", 1, tokenIds.size());
-      long [] result = {0};
+      long result = 0;
       for (TokenIdentifier tokenId : tokenIds) {
         BlockTokenIdentifier id = (BlockTokenIdentifier) tokenId;
         LOG.info("Got: " + id.toString());
         assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
         sm.checkAccess(id, null, block, BlockTokenSecretManager.AccessMode.WRITE);
-        result = id.getBlockIds();
+        result = id.getBlockId();
       }
-      assertEquals("Got more than one block back", 1, result.length);
-      return result[0];
+      return result;
     }
   }
 
@@ -223,28 +221,4 @@ public class TestBlockToken {
     }
   }
 
-  @Test
-  public void collectionOfBlocksActsSanely() {
-    final long[][] testBlockIds = new long [][] {{99l, 7l, -32l, 0l},
-                                                 {},
-                                                 {42l},
-                                                 {-5235l, 2352}};
-    final long [] notBlockIds = new long [] { 32l, 1l, -23423423l};
-
-    for(long [] bids : testBlockIds) {
-      BlockTokenIdentifier bti = new BlockTokenIdentifier("Madame Butterfly", 
-          bids, EnumSet.noneOf(BlockTokenSecretManager.AccessMode.class));
-
-      for(long bid : bids) assertTrue(bti.isBlockIncluded(bid));
-
-      for(long nbid : notBlockIds) assertFalse(bti.isBlockIncluded(nbid));
-
-      // BlockTokenIdentifiers maintain a sorted array of the block Ids.
-      long[] sorted = Arrays.copyOf(bids, bids.length);
-      Arrays.sort(sorted);
-
-      assertTrue(Arrays.toString(bids)+" doesn't equal "+Arrays.toString(sorted), 
-          Arrays.equals(bti.getBlockIds(), sorted));
-    }
-  }
 }



Mime
View raw message