hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From whe...@apache.org
Subject [3/3] hadoop git commit: HDFS-8169. Move LocatedBlocks and related classes to hdfs-client. Contributed by Haohui Mai.
Date Mon, 20 Apr 2015 07:36:45 GMT
HDFS-8169. Move LocatedBlocks and related classes to hdfs-client. Contributed by Haohui Mai.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d3a4d51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d3a4d51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d3a4d51

Branch: refs/heads/branch-2
Commit: 5d3a4d51bde401f2073aeaa3daf15ca2d75b3449
Parents: 490fac3
Author: Haohui Mai <wheat9@apache.org>
Authored: Thu Apr 16 23:13:15 2015 -0700
Committer: Haohui Mai <wheat9@apache.org>
Committed: Mon Apr 20 00:36:19 2015 -0700

----------------------------------------------------------------------
 .../dev-support/findbugsExcludeFile.xml         |   8 +
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |  65 +++
 .../org/apache/hadoop/hdfs/protocol/Block.java  | 243 +++++++++
 .../apache/hadoop/hdfs/protocol/DatanodeID.java | 279 ++++++++++
 .../hadoop/hdfs/protocol/DatanodeInfo.java      | 510 ++++++++++++++++++
 .../hdfs/protocol/DatanodeInfoWithStorage.java  |  67 +++
 .../hadoop/hdfs/protocol/ExtendedBlock.java     | 123 +++++
 .../hdfs/protocol/FsPermissionExtension.java    |  89 ++++
 .../hdfs/protocol/HdfsConstantsClient.java      |   1 +
 .../hadoop/hdfs/protocol/HdfsFileStatus.java    | 271 ++++++++++
 .../hadoop/hdfs/protocol/LocatedBlock.java      | 206 ++++++++
 .../hadoop/hdfs/protocol/LocatedBlocks.java     | 189 +++++++
 .../token/block/BlockTokenIdentifier.java       | 189 +++++++
 .../delegation/DelegationTokenIdentifier.java   | 101 ++++
 .../hadoop/hdfs/web/WebHdfsConstants.java       |  37 ++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   2 +
 .../java/org/apache/hadoop/hdfs/DFSUtil.java    |  15 -
 .../org/apache/hadoop/hdfs/protocol/Block.java  | 243 ---------
 .../apache/hadoop/hdfs/protocol/DatanodeID.java | 279 ----------
 .../hadoop/hdfs/protocol/DatanodeInfo.java      | 512 -------------------
 .../hdfs/protocol/DatanodeInfoWithStorage.java  |  67 ---
 .../hadoop/hdfs/protocol/ExtendedBlock.java     | 123 -----
 .../hdfs/protocol/FsPermissionExtension.java    |  89 ----
 .../hadoop/hdfs/protocol/HdfsFileStatus.java    | 271 ----------
 .../hadoop/hdfs/protocol/LocatedBlock.java      | 217 --------
 .../hadoop/hdfs/protocol/LocatedBlocks.java     | 189 -------
 .../protocol/SnapshottableDirectoryStatus.java  |   3 +-
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java |   3 +-
 .../token/block/BlockTokenIdentifier.java       | 189 -------
 .../delegation/DelegationTokenIdentifier.java   | 102 ----
 .../server/blockmanagement/BlockManager.java    |  19 +-
 .../BlockStoragePolicySuite.java                |   1 -
 .../blockmanagement/DatanodeStorageInfo.java    |   3 +
 .../blockmanagement/HeartbeatManager.java       |  11 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  |   3 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  18 +-
 .../hdfs/server/namenode/FSDirectory.java       |   4 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   7 +-
 .../hdfs/server/namenode/FSEditLogOp.java       |   5 +-
 .../hdfs/server/namenode/FSNamesystem.java      |   5 +-
 .../hadoop/hdfs/server/namenode/INode.java      |  13 +-
 .../hdfs/server/namenode/INodeDirectory.java    |   9 +-
 .../hadoop/hdfs/server/namenode/INodeFile.java  |   8 +-
 .../hadoop/hdfs/server/namenode/INodeMap.java   |   5 +-
 .../snapshot/FileWithSnapshotFeature.java       |   3 +-
 .../web/resources/NamenodeWebHdfsMethods.java   |   6 +-
 .../hadoop/hdfs/tools/StoragePolicyAdmin.java   |   4 +-
 .../apache/hadoop/hdfs/web/JsonUtilClient.java  |   3 +-
 .../hadoop/hdfs/web/SWebHdfsFileSystem.java     |   3 +-
 .../org/apache/hadoop/hdfs/web/TokenAspect.java |  10 +-
 .../hadoop/hdfs/web/WebHdfsConstants.java       |  30 --
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |   5 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java     |   8 +-
 .../namenode/TestNamenodeCapacityReport.java    |  19 +-
 .../apache/hadoop/hdfs/web/TestWebHdfsUrl.java  |   2 +-
 55 files changed, 2475 insertions(+), 2411 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
index 43bc332..478a931 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
@@ -1,2 +1,10 @@
 <FindBugsFilter>
+  <Match>
+    <Or>
+      <Class name="org.apache.hadoop.hdfs.protocol.HdfsFileStatus"/>
+      <Class name="org.apache.hadoop.hdfs.protocol.LocatedBlock"/>
+      <Class name="org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier"/>
+    </Or>
+    <Bug pattern="EI_EXPOSE_REP,EI_EXPOSE_REP2" />
+  </Match>
 </FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
new file mode 100644
index 0000000..2817b66
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.util.StringUtils;
+
+import java.io.UnsupportedEncodingException;
+
+public class DFSUtilClient {
+  /**
+   * Converts a byte array to a string using UTF8 encoding.
+   */
+  public static String bytes2String(byte[] bytes) {
+    return bytes2String(bytes, 0, bytes.length);
+  }
+
+  /** Return used as percentage of capacity */
+  public static float getPercentUsed(long used, long capacity) {
+    return capacity <= 0 ? 100 : (used * 100.0f)/capacity;
+  }
+
+  /** Return remaining as percentage of capacity */
+  public static float getPercentRemaining(long remaining, long capacity) {
+    return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity;
+  }
+
+  /** Convert percentage to a string. */
+  public static String percent2String(double percentage) {
+    return StringUtils.format("%.2f%%", percentage);
+  }
+
+  /**
+   * Decode a specific range of bytes of the given byte array to a string
+   * using UTF8.
+   *
+   * @param bytes The bytes to be decoded into characters
+   * @param offset The index of the first byte to decode
+   * @param length The number of bytes to decode
+   * @return The decoded string
+   */
+  private static String bytes2String(byte[] bytes, int offset, int length) {
+    try {
+      return new String(bytes, offset, length, "UTF8");
+    } catch(UnsupportedEncodingException e) {
+      assert false : "UTF8 encoding is not supported ";
+    }
+    return null;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java
new file mode 100644
index 0000000..11daccc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/Block.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.io.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.*;
+
+/**************************************************
+ * A Block is a Hadoop FS primitive, identified by a
+ * long.
+ *
+ **************************************************/
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class Block implements Writable, Comparable<Block> {
+  public static final String BLOCK_FILE_PREFIX = "blk_";
+  public static final String METADATA_EXTENSION = ".meta";
+  static {                                      // register a ctor
+    WritableFactories.setFactory
+      (Block.class,
+       new WritableFactory() {
+         @Override
+         public Writable newInstance() { return new Block(); }
+       });
+  }
+
+  public static final Pattern blockFilePattern = Pattern
+      .compile(BLOCK_FILE_PREFIX + "(-??\\d++)$");
+  public static final Pattern metaFilePattern = Pattern
+      .compile(BLOCK_FILE_PREFIX + "(-??\\d++)_(\\d++)\\" + METADATA_EXTENSION
+          + "$");
+  public static final Pattern metaOrBlockFilePattern = Pattern
+      .compile(BLOCK_FILE_PREFIX + "(-??\\d++)(_(\\d++)\\" + METADATA_EXTENSION
+          + ")?$");
+
+  public static boolean isBlockFilename(File f) {
+    String name = f.getName();
+    return blockFilePattern.matcher(name).matches();
+  }
+
+  public static long filename2id(String name) {
+    Matcher m = blockFilePattern.matcher(name);
+    return m.matches() ? Long.parseLong(m.group(1)) : 0;
+  }
+
+  public static boolean isMetaFilename(String name) {
+    return metaFilePattern.matcher(name).matches();
+  }
+
+  public static File metaToBlockFile(File metaFile) {
+    return new File(metaFile.getParent(), metaFile.getName().substring(
+        0, metaFile.getName().lastIndexOf('_')));
+  }
+
+  /**
+   * Get generation stamp from the name of the metafile name
+   */
+  public static long getGenerationStamp(String metaFile) {
+    Matcher m = metaFilePattern.matcher(metaFile);
+    return m.matches() ? Long.parseLong(m.group(2))
+        : HdfsConstantsClient.GRANDFATHER_GENERATION_STAMP;
+  }
+
+  /**
+   * Get the blockId from the name of the meta or block file
+   */
+  public static long getBlockId(String metaOrBlockFile) {
+    Matcher m = metaOrBlockFilePattern.matcher(metaOrBlockFile);
+    return m.matches() ? Long.parseLong(m.group(1)) : 0;
+  }
+
+  private long blockId;
+  private long numBytes;
+  private long generationStamp;
+
+  public Block() {this(0, 0, 0);}
+
+  public Block(final long blkid, final long len, final long generationStamp) {
+    set(blkid, len, generationStamp);
+  }
+
+  public Block(final long blkid) {
+    this(blkid, 0, HdfsConstantsClient.GRANDFATHER_GENERATION_STAMP);
+  }
+
+  public Block(Block blk) {
+    this(blk.blockId, blk.numBytes, blk.generationStamp);
+  }
+
+  /**
+   * Find the blockid from the given filename
+   */
+  public Block(File f, long len, long genstamp) {
+    this(filename2id(f.getName()), len, genstamp);
+  }
+
+  public void set(long blkid, long len, long genStamp) {
+    this.blockId = blkid;
+    this.numBytes = len;
+    this.generationStamp = genStamp;
+  }
+  /**
+   */
+  public long getBlockId() {
+    return blockId;
+  }
+
+  public void setBlockId(long bid) {
+    blockId = bid;
+  }
+
+  /**
+   */
+  public String getBlockName() {
+    return BLOCK_FILE_PREFIX + String.valueOf(blockId);
+  }
+
+  /**
+   */
+  public long getNumBytes() {
+    return numBytes;
+  }
+  public void setNumBytes(long len) {
+    this.numBytes = len;
+  }
+
+  public long getGenerationStamp() {
+    return generationStamp;
+  }
+
+  public void setGenerationStamp(long stamp) {
+    generationStamp = stamp;
+  }
+
+  /**
+   */
+  @Override
+  public String toString() {
+    return getBlockName() + "_" + getGenerationStamp();
+  }
+
+  public void appendStringTo(StringBuilder sb) {
+    sb.append(BLOCK_FILE_PREFIX)
+      .append(blockId)
+      .append("_")
+      .append(getGenerationStamp());
+  }
+
+
+  /////////////////////////////////////
+  // Writable
+  /////////////////////////////////////
+  @Override // Writable
+  public void write(DataOutput out) throws IOException {
+    writeHelper(out);
+  }
+
+  @Override // Writable
+  public void readFields(DataInput in) throws IOException {
+    readHelper(in);
+  }
+
+  final void writeHelper(DataOutput out) throws IOException {
+    out.writeLong(blockId);
+    out.writeLong(numBytes);
+    out.writeLong(generationStamp);
+  }
+
+  final void readHelper(DataInput in) throws IOException {
+    this.blockId = in.readLong();
+    this.numBytes = in.readLong();
+    this.generationStamp = in.readLong();
+    if (numBytes < 0) {
+      throw new IOException("Unexpected block size: " + numBytes);
+    }
+  }
+
+  // write only the identifier part of the block
+  public void writeId(DataOutput out) throws IOException {
+    out.writeLong(blockId);
+    out.writeLong(generationStamp);
+  }
+
+  // Read only the identifier part of the block
+  public void readId(DataInput in) throws IOException {
+    this.blockId = in.readLong();
+    this.generationStamp = in.readLong();
+  }
+
+  @Override // Comparable
+  public int compareTo(Block b) {
+    return blockId < b.blockId ? -1 :
+           blockId > b.blockId ? 1 : 0;
+  }
+
+  @Override // Object
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof Block)) {
+      return false;
+    }
+    return compareTo((Block)o) == 0;
+  }
+
+  /**
+   * @return true if the two blocks have the same block ID and the same
+   * generation stamp, or if both blocks are null.
+   */
+  public static boolean matchingIdAndGenStamp(Block a, Block b) {
+    if (a == b) return true; // same block, or both null
+    if (a == null || b == null) return false; // only one null
+    return a.blockId == b.blockId &&
+           a.generationStamp == b.generationStamp;
+  }
+
+  @Override // Object
+  public int hashCode() {
+    //GenerationStamp is IRRELEVANT and should not be used here
+    return (int)(blockId^(blockId>>>32));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
new file mode 100644
index 0000000..6d72285
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
@@ -0,0 +1,279 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This class represents the primary identifier for a Datanode.
+ * Datanodes are identified by how they can be contacted (hostname
+ * and ports) and their storage ID, a unique number that associates
+ * the Datanodes blocks with a particular Datanode.
+ *
+ * {@link DatanodeInfo#getName()} should be used to get the network
+ * location (for topology) of a datanode, instead of using
+ * {@link DatanodeID#getXferAddr()} here. Helpers are defined below
+ * for each context in which a DatanodeID is used.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DatanodeID implements Comparable<DatanodeID> {
+  public static final DatanodeID[] EMPTY_ARRAY = {};
+
+  private String ipAddr;     // IP address
+  private String hostName;   // hostname claimed by datanode
+  private String peerHostName; // hostname from the actual connection
+  private int xferPort;      // data streaming port
+  private int infoPort;      // info server port
+  private int infoSecurePort; // info server port
+  private int ipcPort;       // IPC server port
+  private String xferAddr;
+
+  /**
+   * UUID identifying a given datanode. For upgraded Datanodes this is the
+   * same as the StorageID that was previously used by this Datanode.
+   * For newly formatted Datanodes it is a UUID.
+   */
+  private final String datanodeUuid;
+
+  public DatanodeID(DatanodeID from) {
+    this(from.getDatanodeUuid(), from);
+  }
+
+  @VisibleForTesting
+  public DatanodeID(String datanodeUuid, DatanodeID from) {
+    this(from.getIpAddr(),
+        from.getHostName(),
+        datanodeUuid,
+        from.getXferPort(),
+        from.getInfoPort(),
+        from.getInfoSecurePort(),
+        from.getIpcPort());
+    this.peerHostName = from.getPeerHostName();
+  }
+
+  /**
+   * Create a DatanodeID
+   * @param ipAddr IP
+   * @param hostName hostname
+   * @param datanodeUuid data node ID, UUID for new Datanodes, may be the
+   *                     storage ID for pre-UUID datanodes. NULL if unknown
+   *                     e.g. if this is a new datanode. A new UUID will
+   *                     be assigned by the namenode.
+   * @param xferPort data transfer port
+   * @param infoPort info server port
+   * @param ipcPort ipc server port
+   */
+  public DatanodeID(String ipAddr, String hostName, String datanodeUuid,
+      int xferPort, int infoPort, int infoSecurePort, int ipcPort) {
+    setIpAndXferPort(ipAddr, xferPort);
+    this.hostName = hostName;
+    this.datanodeUuid = checkDatanodeUuid(datanodeUuid);
+    this.infoPort = infoPort;
+    this.infoSecurePort = infoSecurePort;
+    this.ipcPort = ipcPort;
+  }
+
+  public void setIpAddr(String ipAddr) {
+    //updated during registration, preserve former xferPort
+    setIpAndXferPort(ipAddr, xferPort);
+  }
+
+  private void setIpAndXferPort(String ipAddr, int xferPort) {
+    // build xferAddr string to reduce cost of frequent use
+    this.ipAddr = ipAddr;
+    this.xferPort = xferPort;
+    this.xferAddr = ipAddr + ":" + xferPort;
+  }
+
+  public void setPeerHostName(String peerHostName) {
+    this.peerHostName = peerHostName;
+  }
+
+  /**
+   * @return data node ID.
+   */
+  public String getDatanodeUuid() {
+    return datanodeUuid;
+  }
+
+  private String checkDatanodeUuid(String uuid) {
+    if (uuid == null || uuid.isEmpty()) {
+      return null;
+    } else {
+      return uuid;
+    }
+  }
+
+  /**
+   * @return ipAddr;
+   */
+  public String getIpAddr() {
+    return ipAddr;
+  }
+
+  /**
+   * @return hostname
+   */
+  public String getHostName() {
+    return hostName;
+  }
+
+  /**
+   * @return hostname from the actual connection
+   */
+  public String getPeerHostName() {
+    return peerHostName;
+  }
+
+  /**
+   * @return IP:xferPort string
+   */
+  public String getXferAddr() {
+    return xferAddr;
+  }
+
+  /**
+   * @return IP:ipcPort string
+   */
+  private String getIpcAddr() {
+    return ipAddr + ":" + ipcPort;
+  }
+
+  /**
+   * @return IP:infoPort string
+   */
+  public String getInfoAddr() {
+    return ipAddr + ":" + infoPort;
+  }
+
+  /**
+   * @return IP:infoPort string
+   */
+  public String getInfoSecureAddr() {
+    return ipAddr + ":" + infoSecurePort;
+  }
+
+  /**
+   * @return hostname:xferPort
+   */
+  public String getXferAddrWithHostname() {
+    return hostName + ":" + xferPort;
+  }
+
+  /**
+   * @return hostname:ipcPort
+   */
+  private String getIpcAddrWithHostname() {
+    return hostName + ":" + ipcPort;
+  }
+
+  /**
+   * @param useHostname true to use the DN hostname, use the IP otherwise
+   * @return name:xferPort
+   */
+  public String getXferAddr(boolean useHostname) {
+    return useHostname ? getXferAddrWithHostname() : getXferAddr();
+  }
+
+  /**
+   * @param useHostname true to use the DN hostname, use the IP otherwise
+   * @return name:ipcPort
+   */
+  public String getIpcAddr(boolean useHostname) {
+    return useHostname ? getIpcAddrWithHostname() : getIpcAddr();
+  }
+
+  /**
+   * @return xferPort (the port for data streaming)
+   */
+  public int getXferPort() {
+    return xferPort;
+  }
+
+  /**
+   * @return infoPort (the port at which the HTTP server bound to)
+   */
+  public int getInfoPort() {
+    return infoPort;
+  }
+
+  /**
+   * @return infoSecurePort (the port at which the HTTPS server bound to)
+   */
+  public int getInfoSecurePort() {
+    return infoSecurePort;
+  }
+
+  /**
+   * @return ipcPort (the port at which the IPC server bound to)
+   */
+  public int getIpcPort() {
+    return ipcPort;
+  }
+
+  @Override
+  public boolean equals(Object to) {
+    if (this == to) {
+      return true;
+    }
+    if (!(to instanceof DatanodeID)) {
+      return false;
+    }
+    return (getXferAddr().equals(((DatanodeID)to).getXferAddr()) &&
+        datanodeUuid.equals(((DatanodeID)to).getDatanodeUuid()));
+  }
+
+  @Override
+  public int hashCode() {
+    return datanodeUuid.hashCode();
+  }
+
+  @Override
+  public String toString() {
+    return getXferAddr();
+  }
+
+  /**
+   * Update fields when a new registration request comes in.
+   * Note that this does not update storageID.
+   */
+  public void updateRegInfo(DatanodeID nodeReg) {
+    setIpAndXferPort(nodeReg.getIpAddr(), nodeReg.getXferPort());
+    hostName = nodeReg.getHostName();
+    peerHostName = nodeReg.getPeerHostName();
+    infoPort = nodeReg.getInfoPort();
+    infoSecurePort = nodeReg.getInfoSecurePort();
+    ipcPort = nodeReg.getIpcPort();
+  }
+
+  /**
+   * Compare based on data transfer address.
+   *
+   * @param that datanode to compare with
+   * @return as specified by Comparable
+   */
+  @Override
+  public int compareTo(DatanodeID that) {
+    return getXferAddr().compareTo(that.getXferAddr());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
new file mode 100644
index 0000000..986cd5c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -0,0 +1,510 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
+
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.DFSUtilClient.percent2String;
+
+/**
+ * This class extends the primary identifier of a Datanode with ephemeral
+ * state, eg usage information, current administrative state, and the
+ * network location that is communicated to clients.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DatanodeInfo extends DatanodeID implements Node {
+  private long capacity;
+  private long dfsUsed;
+  private long remaining;
+  private long blockPoolUsed;
+  private long cacheCapacity;
+  private long cacheUsed;
+  private long lastUpdate;
+  private long lastUpdateMonotonic;
+  private int xceiverCount;
+  private String location = NetworkTopology.DEFAULT_RACK;
+  private String softwareVersion;
+  private List<String> dependentHostNames = new LinkedList<String>();
+
+
+  // Datanode administrative states
+  public enum AdminStates {
+    NORMAL("In Service"),
+    DECOMMISSION_INPROGRESS("Decommission In Progress"),
+    DECOMMISSIONED("Decommissioned");
+
+    final String value;
+
+    AdminStates(final String v) {
+      this.value = v;
+    }
+
+    @Override
+    public String toString() {
+      return value;
+    }
+
+    public static AdminStates fromValue(final String value) {
+      for (AdminStates as : AdminStates.values()) {
+        if (as.value.equals(value)) return as;
+      }
+      return NORMAL;
+    }
+  }
+
+  protected AdminStates adminState;
+
+  public DatanodeInfo(DatanodeInfo from) {
+    super(from);
+    this.capacity = from.getCapacity();
+    this.dfsUsed = from.getDfsUsed();
+    this.remaining = from.getRemaining();
+    this.blockPoolUsed = from.getBlockPoolUsed();
+    this.cacheCapacity = from.getCacheCapacity();
+    this.cacheUsed = from.getCacheUsed();
+    this.lastUpdate = from.getLastUpdate();
+    this.lastUpdateMonotonic = from.getLastUpdateMonotonic();
+    this.xceiverCount = from.getXceiverCount();
+    this.location = from.getNetworkLocation();
+    this.adminState = from.getAdminState();
+  }
+
+  public DatanodeInfo(DatanodeID nodeID) {
+    super(nodeID);
+    this.capacity = 0L;
+    this.dfsUsed = 0L;
+    this.remaining = 0L;
+    this.blockPoolUsed = 0L;
+    this.cacheCapacity = 0L;
+    this.cacheUsed = 0L;
+    this.lastUpdate = 0L;
+    this.lastUpdateMonotonic = 0L;
+    this.xceiverCount = 0;
+    this.adminState = null;
+  }
+
+  public DatanodeInfo(DatanodeID nodeID, String location) {
+    this(nodeID);
+    this.location = location;
+  }
+
+  public DatanodeInfo(DatanodeID nodeID, String location,
+      final long capacity, final long dfsUsed, final long remaining,
+      final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+      final long lastUpdate, final long lastUpdateMonotonic,
+      final int xceiverCount, final AdminStates adminState) {
+    this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(),
+        nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getInfoSecurePort(),
+        nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed,
+        cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
+        xceiverCount, location, adminState);
+  }
+
+  /** Constructor */
+  public DatanodeInfo(final String ipAddr, final String hostName,
+      final String datanodeUuid, final int xferPort, final int infoPort,
+      final int infoSecurePort, final int ipcPort,
+      final long capacity, final long dfsUsed, final long remaining,
+      final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+      final long lastUpdate, final long lastUpdateMonotonic,
+      final int xceiverCount, final String networkLocation,
+      final AdminStates adminState) {
+    super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
+            infoSecurePort, ipcPort);
+    this.capacity = capacity;
+    this.dfsUsed = dfsUsed;
+    this.remaining = remaining;
+    this.blockPoolUsed = blockPoolUsed;
+    this.cacheCapacity = cacheCapacity;
+    this.cacheUsed = cacheUsed;
+    this.lastUpdate = lastUpdate;
+    this.lastUpdateMonotonic = lastUpdateMonotonic;
+    this.xceiverCount = xceiverCount;
+    this.location = networkLocation;
+    this.adminState = adminState;
+  }
+
+  /** Network location name */
+  @Override
+  public String getName() {
+    return getXferAddr();
+  }
+
+  /** The raw capacity. */
+  public long getCapacity() { return capacity; }
+
+  /** The used space by the data node. */
+  public long getDfsUsed() { return dfsUsed; }
+
+  /** The used space by the block pool on data node. */
+  public long getBlockPoolUsed() { return blockPoolUsed; }
+
+  /** The used space by the data node. */
+  public long getNonDfsUsed() {
+    long nonDFSUsed = capacity - dfsUsed - remaining;
+    return nonDFSUsed < 0 ? 0 : nonDFSUsed;
+  }
+
+  /** The used space by the data node as percentage of present capacity */
+  public float getDfsUsedPercent() {
+    return DFSUtilClient.getPercentUsed(dfsUsed, capacity);
+  }
+
+  /** The raw free space. */
+  public long getRemaining() { return remaining; }
+
+  /** Used space by the block pool as percentage of present capacity */
+  public float getBlockPoolUsedPercent() {
+    return DFSUtilClient.getPercentUsed(blockPoolUsed, capacity);
+  }
+
+  /** The remaining space as percentage of configured capacity. */
+  public float getRemainingPercent() {
+    return DFSUtilClient.getPercentRemaining(remaining, capacity);
+  }
+
+  /**
+   * @return Amount of cache capacity in bytes
+   */
+  public long getCacheCapacity() {
+    return cacheCapacity;
+  }
+
+  /**
+   * @return Amount of cache used in bytes
+   */
+  public long getCacheUsed() {
+    return cacheUsed;
+  }
+
+  /**
+   * @return Cache used as a percentage of the datanode's total cache capacity
+   */
+  public float getCacheUsedPercent() {
+    return DFSUtilClient.getPercentUsed(cacheUsed, cacheCapacity);
+  }
+
+  /**
+   * @return Amount of cache remaining in bytes
+   */
+  public long getCacheRemaining() {
+    return cacheCapacity - cacheUsed;
+  }
+
+  /**
+   * @return Cache remaining as a percentage of the datanode's total cache
+   * capacity
+   */
+  public float getCacheRemainingPercent() {
+    return DFSUtilClient.getPercentRemaining(getCacheRemaining(), cacheCapacity);
+  }
+
+  /**
+   * Get the last update timestamp.
+   * Return value is suitable for Date conversion.
+   */
+  public long getLastUpdate() { return lastUpdate; }
+
+  /**
+   * The time when this information was accurate. <br>
+   * Ps: So return value is ideal for calculation of time differences.
+   * Should not be used to convert to Date.
+   */
+  public long getLastUpdateMonotonic() { return lastUpdateMonotonic;}
+
+  /**
+   * Set lastUpdate monotonic time
+   */
+  public void setLastUpdateMonotonic(long lastUpdateMonotonic) {
+    this.lastUpdateMonotonic = lastUpdateMonotonic;
+  }
+
+  /** number of active connections */
+  public int getXceiverCount() { return xceiverCount; }
+
+  /** Sets raw capacity. */
+  public void setCapacity(long capacity) {
+    this.capacity = capacity;
+  }
+
+  /** Sets the used space for the datanode. */
+  public void setDfsUsed(long dfsUsed) {
+    this.dfsUsed = dfsUsed;
+  }
+
+  /** Sets raw free space. */
+  public void setRemaining(long remaining) {
+    this.remaining = remaining;
+  }
+
+  /** Sets block pool used space */
+  public void setBlockPoolUsed(long bpUsed) {
+    this.blockPoolUsed = bpUsed;
+  }
+
+  /** Sets cache capacity. */
+  public void setCacheCapacity(long cacheCapacity) {
+    this.cacheCapacity = cacheCapacity;
+  }
+
+  /** Sets cache used. */
+  public void setCacheUsed(long cacheUsed) {
+    this.cacheUsed = cacheUsed;
+  }
+
+  /** Sets time when this information was accurate. */
+  public void setLastUpdate(long lastUpdate) {
+    this.lastUpdate = lastUpdate;
+  }
+
+  /** Sets number of active connections */
+  public void setXceiverCount(int xceiverCount) {
+    this.xceiverCount = xceiverCount;
+  }
+
+  /** network location */
+  public synchronized String getNetworkLocation() {return location;}
+
+  /** Sets the network location */
+  public synchronized void setNetworkLocation(String location) {
+    this.location = NodeBase.normalize(location);
+  }
+
+  /** Add a hostname to a list of network dependencies */
+  public void addDependentHostName(String hostname) {
+    dependentHostNames.add(hostname);
+  }
+
+  /** List of Network dependencies */
+  public List<String> getDependentHostNames() {
+    return dependentHostNames;
+  }
+
+  /** Sets the network dependencies */
+  public void setDependentHostNames(List<String> dependencyList) {
+    dependentHostNames = dependencyList;
+  }
+
+  /** A formatted string for reporting the status of the DataNode. */
+  public String getDatanodeReport() {
+    StringBuilder buffer = new StringBuilder();
+    long c = getCapacity();
+    long r = getRemaining();
+    long u = getDfsUsed();
+    long nonDFSUsed = getNonDfsUsed();
+    float usedPercent = getDfsUsedPercent();
+    float remainingPercent = getRemainingPercent();
+    long cc = getCacheCapacity();
+    long cr = getCacheRemaining();
+    long cu = getCacheUsed();
+    float cacheUsedPercent = getCacheUsedPercent();
+    float cacheRemainingPercent = getCacheRemainingPercent();
+    String lookupName = NetUtils.getHostNameOfIP(getName());
+
+    buffer.append("Name: "+ getName());
+    if (lookupName != null) {
+      buffer.append(" (" + lookupName + ")");
+    }
+    buffer.append("\n");
+    buffer.append("Hostname: " + getHostName() + "\n");
+
+    if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
+      buffer.append("Rack: "+location+"\n");
+    }
+    buffer.append("Decommission Status : ");
+    if (isDecommissioned()) {
+      buffer.append("Decommissioned\n");
+    } else if (isDecommissionInProgress()) {
+      buffer.append("Decommission in progress\n");
+    } else {
+      buffer.append("Normal\n");
+    }
+    buffer.append("Configured Capacity: "+c+" ("+StringUtils.byteDesc(c)+")"+"\n");
+    buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n");
+    buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n");
+    buffer.append("DFS Remaining: " +r+ " ("+StringUtils.byteDesc(r)+")"+"\n");
+    buffer.append("DFS Used%: "+percent2String(usedPercent) + "\n");
+    buffer.append("DFS Remaining%: "+percent2String(remainingPercent) + "\n");
+    buffer.append("Configured Cache Capacity: "+cc+" ("+StringUtils.byteDesc(cc)+")"+"\n");
+    buffer.append("Cache Used: "+cu+" ("+StringUtils.byteDesc(cu)+")"+"\n");
+    buffer.append("Cache Remaining: " +cr+ " ("+StringUtils.byteDesc(cr)+")"+"\n");
+    buffer.append("Cache Used%: "+percent2String(cacheUsedPercent) + "\n");
+    buffer.append("Cache Remaining%: "+percent2String(cacheRemainingPercent) + "\n");
+    buffer.append("Xceivers: "+getXceiverCount()+"\n");
+    buffer.append("Last contact: "+new Date(lastUpdate)+"\n");
+    return buffer.toString();
+  }
+
+  /** A formatted string for printing the status of the DataNode. */
+  public String dumpDatanode() {
+    StringBuilder buffer = new StringBuilder();
+    long c = getCapacity();
+    long r = getRemaining();
+    long u = getDfsUsed();
+    long cc = getCacheCapacity();
+    long cr = getCacheRemaining();
+    long cu = getCacheUsed();
+    buffer.append(getName());
+    if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
+      buffer.append(" "+location);
+    }
+    if (isDecommissioned()) {
+      buffer.append(" DD");
+    } else if (isDecommissionInProgress()) {
+      buffer.append(" DP");
+    } else {
+      buffer.append(" IN");
+    }
+    buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
+    buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
+    buffer.append(" " + percent2String(u/(double)c));
+    buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
+    buffer.append(" " + cc + "(" + StringUtils.byteDesc(cc)+")");
+    buffer.append(" " + cu + "(" + StringUtils.byteDesc(cu)+")");
+    buffer.append(" " + percent2String(cu/(double)cc));
+    buffer.append(" " + cr + "(" + StringUtils.byteDesc(cr)+")");
+    buffer.append(" " + new Date(lastUpdate));
+    return buffer.toString();
+  }
+
+  /**
+   * Start decommissioning a node.
+   * old state.
+   */
+  public void startDecommission() {
+    adminState = AdminStates.DECOMMISSION_INPROGRESS;
+  }
+
+  /**
+   * Stop decommissioning a node.
+   * old state.
+   */
+  public void stopDecommission() {
+    adminState = null;
+  }
+
+  /**
+   * Returns true if the node is in the process of being decommissioned
+   */
+  public boolean isDecommissionInProgress() {
+    return adminState == AdminStates.DECOMMISSION_INPROGRESS;
+  }
+
+  /**
+   * Returns true if the node has been decommissioned.
+   */
+  public boolean isDecommissioned() {
+    return adminState == AdminStates.DECOMMISSIONED;
+  }
+
+  /**
+   * Sets the admin state to indicate that decommission is complete.
+   */
+  public void setDecommissioned() {
+    adminState = AdminStates.DECOMMISSIONED;
+  }
+
+  /**
+   * Retrieves the admin state of this node.
+   */
+  public AdminStates getAdminState() {
+    if (adminState == null) {
+      return AdminStates.NORMAL;
+    }
+    return adminState;
+  }
+
+  /**
+   * Check if the datanode is in stale state. Here if
+   * the namenode has not received heartbeat msg from a
+   * datanode for more than staleInterval,
+   * the datanode will be treated as stale node.
+   *
+   * @param staleInterval
+   *          the time interval for marking the node as stale. If the last
+   *          update time is beyond the given time interval, the node will be
+   *          marked as stale.
+   * @return true if the node is stale
+   */
+  public boolean isStale(long staleInterval) {
+    return (Time.monotonicNow() - lastUpdateMonotonic) >= staleInterval;
+  }
+
+  /**
+   * Sets the admin state of this node.
+   */
+  protected void setAdminState(AdminStates newState) {
+    if (newState == AdminStates.NORMAL) {
+      adminState = null;
+    }
+    else {
+      adminState = newState;
+    }
+  }
+
+  private transient int level; //which level of the tree the node resides
+  private transient Node parent; //its parent
+
+  /** Return this node's parent */
+  @Override
+  public Node getParent() { return parent; }
+  @Override
+  public void setParent(Node parent) {this.parent = parent;}
+
+  /** Return this node's level in the tree.
+   * E.g. the root of a tree returns 0 and its children return 1
+   */
+  @Override
+  public int getLevel() { return level; }
+  @Override
+  public void setLevel(int level) {this.level = level;}
+
+  @Override
+  public int hashCode() {
+    // Super implementation is sufficient
+    return super.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    // Sufficient to use super equality as datanodes are uniquely identified
+    // by DatanodeID
+    return (this == obj) || super.equals(obj);
+  }
+
+  public String getSoftwareVersion() {
+    return softwareVersion;
+  }
+
+  public void setSoftwareVersion(String softwareVersion) {
+    this.softwareVersion = softwareVersion;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java
new file mode 100644
index 0000000..18c940b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DatanodeInfoWithStorage extends DatanodeInfo {
+  private final String storageID;
+  private final StorageType storageType;
+
+  public DatanodeInfoWithStorage(DatanodeInfo from, String storageID,
+                                 StorageType storageType) {
+    super(from);
+    this.storageID = storageID;
+    this.storageType = storageType;
+    setSoftwareVersion(from.getSoftwareVersion());
+    setDependentHostNames(from.getDependentHostNames());
+    setLevel(from.getLevel());
+    setParent(from.getParent());
+  }
+
+  public String getStorageID() {
+    return storageID;
+  }
+
+  public StorageType getStorageType() {
+    return storageType;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    // allows this class to be used interchangeably with DatanodeInfo
+    return super.equals(o);
+  }
+
+  @Override
+  public int hashCode() {
+    // allows this class to be used interchangeably with DatanodeInfo
+    return super.hashCode();
+  }
+
+  @Override
+  public String toString() {
+    return "DatanodeInfoWithStorage[" + super.toString() + "," + storageID +
+        "," + storageType + "]";
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
new file mode 100644
index 0000000..af24909
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Identifies a Block uniquely across the block pools
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ExtendedBlock {
+  private String poolId;
+  private Block block;
+
+  public ExtendedBlock() {
+    this(null, 0, 0, 0);
+  }
+
+  public ExtendedBlock(final ExtendedBlock b) {
+    this(b.poolId, new Block(b.block));
+  }
+
+  public ExtendedBlock(final String poolId, final long blockId) {
+    this(poolId, blockId, 0, 0);
+  }
+
+  public ExtendedBlock(String poolId, Block b) {
+    this.poolId = poolId;
+    this.block = b;
+  }
+
+  public ExtendedBlock(final String poolId, final long blkid, final long len,
+      final long genstamp) {
+    this.poolId = poolId;
+    block = new Block(blkid, len, genstamp);
+  }
+
+  public String getBlockPoolId() {
+    return poolId;
+  }
+
+  /** Returns the block file name for the block */
+  public String getBlockName() {
+    return block.getBlockName();
+  }
+
+  public long getNumBytes() {
+    return block.getNumBytes();
+  }
+
+  public long getBlockId() {
+    return block.getBlockId();
+  }
+
+  public long getGenerationStamp() {
+    return block.getGenerationStamp();
+  }
+
+  public void setBlockId(final long bid) {
+    block.setBlockId(bid);
+  }
+
+  public void setGenerationStamp(final long genStamp) {
+    block.setGenerationStamp(genStamp);
+  }
+
+  public void setNumBytes(final long len) {
+    block.setNumBytes(len);
+  }
+
+  public void set(String poolId, Block blk) {
+    this.poolId = poolId;
+    this.block = blk;
+  }
+
+  public static Block getLocalBlock(final ExtendedBlock b) {
+    return b == null ? null : b.getLocalBlock();
+  }
+
+  public Block getLocalBlock() {
+    return block;
+  }
+
+  @Override // Object
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof ExtendedBlock)) {
+      return false;
+    }
+    ExtendedBlock b = (ExtendedBlock)o;
+    return b.block.equals(block) && b.poolId.equals(poolId);
+  }
+
+  @Override // Object
+  public int hashCode() {
+    int result = 31 + poolId.hashCode();
+    return (31 * result + block.hashCode());
+  }
+
+  @Override // Object
+  public String toString() {
+    return poolId + ":" + block;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
new file mode 100644
index 0000000..f74472d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * HDFS permission subclass used to indicate an ACL is present and/or that the
+ * underlying file/dir is encrypted. The ACL/encrypted bits are not visible
+ * directly to users of {@link FsPermission} serialization.  This is
+ * done for backwards compatibility in case any existing clients assume the
+ * value of FsPermission is in a particular range.
+ */
+@InterfaceAudience.Private
+public class FsPermissionExtension extends FsPermission {
+  private final static short ACL_BIT = 1 << 12;
+  private final static short ENCRYPTED_BIT = 1 << 13;
+  private final boolean aclBit;
+  private final boolean encryptedBit;
+
+  /**
+   * Constructs a new FsPermissionExtension based on the given FsPermission.
+   *
+   * @param perm FsPermission containing permission bits
+   */
+  public FsPermissionExtension(FsPermission perm, boolean hasAcl,
+      boolean isEncrypted) {
+    super(perm.toShort());
+    aclBit = hasAcl;
+    encryptedBit = isEncrypted;
+  }
+
+  /**
+   * Creates a new FsPermissionExtension by calling the base class constructor.
+   *
+   * @param perm short containing permission bits
+   */
+  public FsPermissionExtension(short perm) {
+    super(perm);
+    aclBit = (perm & ACL_BIT) != 0;
+    encryptedBit = (perm & ENCRYPTED_BIT) != 0;
+  }
+
+  @Override
+  public short toExtendedShort() {
+    return (short)(toShort() |
+        (aclBit ? ACL_BIT : 0) | (encryptedBit ? ENCRYPTED_BIT : 0));
+  }
+
+  @Override
+  public boolean getAclBit() {
+    return aclBit;
+  }
+
+  @Override
+  public boolean getEncryptedBit() {
+    return encryptedBit;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    // This intentionally delegates to the base class.  This is only overridden
+    // to suppress a FindBugs warning.
+    return super.equals(o);
+  }
+
+  @Override
+  public int hashCode() {
+    // This intentionally delegates to the base class.  This is only overridden
+    // to suppress a FindBugs warning.
+    return super.hashCode();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java
index aa5f1f9..40c3203 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstantsClient.java
@@ -31,4 +31,5 @@ public interface HdfsConstantsClient {
    * uses GRANDFATHER_INODE_ID for backward compatibility.
    */
   long GRANDFATHER_INODE_ID = 0;
+  byte BLOCK_STORAGE_POLICY_ID_UNSPECIFIED = 0;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
new file mode 100644
index 0000000..34f429a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+
+/** Interface that represents the over the wire information for a file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class HdfsFileStatus {
+
+  private final byte[] path;  // local name of the inode that's encoded in java UTF8
+  private final byte[] symlink; // symlink target encoded in java UTF8 or null
+  private final long length;
+  private final boolean isdir;
+  private final short block_replication;
+  private final long blocksize;
+  private final long modification_time;
+  private final long access_time;
+  private final FsPermission permission;
+  private final String owner;
+  private final String group;
+  private final long fileId;
+
+  private final FileEncryptionInfo feInfo;
+
+  // Used by dir, not including dot and dotdot. Always zero for a regular file.
+  private final int childrenNum;
+  private final byte storagePolicy;
+
+  public static final byte[] EMPTY_NAME = new byte[0];
+
+  /**
+   * Constructor
+   * @param length the number of bytes the file has
+   * @param isdir if the path is a directory
+   * @param block_replication the replication factor
+   * @param blocksize the block size
+   * @param modification_time modification time
+   * @param access_time access time
+   * @param permission permission
+   * @param owner the owner of the path
+   * @param group the group of the path
+   * @param path the local name in java UTF8 encoding the same as that in-memory
+   * @param fileId the file id
+   * @param feInfo the file's encryption info
+   */
+  public HdfsFileStatus(long length, boolean isdir, int block_replication,
+      long blocksize, long modification_time, long access_time,
+      FsPermission permission, String owner, String group, byte[] symlink,
+      byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo,
+      byte storagePolicy) {
+    this.length = length;
+    this.isdir = isdir;
+    this.block_replication = (short)block_replication;
+    this.blocksize = blocksize;
+    this.modification_time = modification_time;
+    this.access_time = access_time;
+    this.permission = (permission == null) ?
+        ((isdir || symlink!=null) ?
+            FsPermission.getDefault() :
+            FsPermission.getFileDefault()) :
+        permission;
+    this.owner = (owner == null) ? "" : owner;
+    this.group = (group == null) ? "" : group;
+    this.symlink = symlink;
+    this.path = path;
+    this.fileId = fileId;
+    this.childrenNum = childrenNum;
+    this.feInfo = feInfo;
+    this.storagePolicy = storagePolicy;
+  }
+
+  /**
+   * Get the length of this file, in bytes.
+   * @return the length of this file, in bytes.
+   */
+  public final long getLen() {
+    return length;
+  }
+
+  /**
+   * Is this a directory?
+   * @return true if this is a directory
+   */
+  public final boolean isDir() {
+    return isdir;
+  }
+
+  /**
+   * Is this a symbolic link?
+   * @return true if this is a symbolic link
+   */
+  public boolean isSymlink() {
+    return symlink != null;
+  }
+
+  /**
+   * Get the block size of the file.
+   * @return the number of bytes
+   */
+  public final long getBlockSize() {
+    return blocksize;
+  }
+
+  /**
+   * Get the replication factor of a file.
+   * @return the replication factor of a file.
+   */
+  public final short getReplication() {
+    return block_replication;
+  }
+
+  /**
+   * Get the modification time of the file.
+   * @return the modification time of file in milliseconds since January 1, 1970 UTC.
+   */
+  public final long getModificationTime() {
+    return modification_time;
+  }
+
+  /**
+   * Get the access time of the file.
+   * @return the access time of file in milliseconds since January 1, 1970 UTC.
+   */
+  public final long getAccessTime() {
+    return access_time;
+  }
+
+  /**
+   * Get FsPermission associated with the file.
+   * @return permssion
+   */
+  public final FsPermission getPermission() {
+    return permission;
+  }
+
+  /**
+   * Get the owner of the file.
+   * @return owner of the file
+   */
+  public final String getOwner() {
+    return owner;
+  }
+
+  /**
+   * Get the group associated with the file.
+   * @return group for the file.
+   */
+  public final String getGroup() {
+    return group;
+  }
+
+  /**
+   * Check if the local name is empty
+   * @return true if the name is empty
+   */
+  public final boolean isEmptyLocalName() {
+    return path.length == 0;
+  }
+
+  /**
+   * Get the string representation of the local name
+   * @return the local name in string
+   */
+  public final String getLocalName() {
+    return DFSUtilClient.bytes2String(path);
+  }
+
+  /**
+   * Get the Java UTF8 representation of the local name
+   * @return the local name in java UTF8
+   */
+  public final byte[] getLocalNameInBytes() {
+    return path;
+  }
+
+  /**
+   * Get the string representation of the full path name
+   * @param parent the parent path
+   * @return the full path in string
+   */
+  public final String getFullName(final String parent) {
+    if (isEmptyLocalName()) {
+      return parent;
+    }
+
+    StringBuilder fullName = new StringBuilder(parent);
+    if (!parent.endsWith(Path.SEPARATOR)) {
+      fullName.append(Path.SEPARATOR);
+    }
+    fullName.append(getLocalName());
+    return fullName.toString();
+  }
+
+  /**
+   * Get the full path
+   * @param parent the parent path
+   * @return the full path
+   */
+  public final Path getFullPath(final Path parent) {
+    if (isEmptyLocalName()) {
+      return parent;
+    }
+
+    return new Path(parent, getLocalName());
+  }
+
+  /**
+   * Get the string representation of the symlink.
+   * @return the symlink as a string.
+   */
+  public final String getSymlink() {
+    return DFSUtilClient.bytes2String(symlink);
+  }
+
+  public final byte[] getSymlinkInBytes() {
+    return symlink;
+  }
+
+  public final long getFileId() {
+    return fileId;
+  }
+
+  public final FileEncryptionInfo getFileEncryptionInfo() {
+    return feInfo;
+  }
+
+  public final int getChildrenNum() {
+    return childrenNum;
+  }
+
+  /** @return the storage policy id */
+  public final byte getStoragePolicy() {
+    return storagePolicy;
+  }
+
+  public final FileStatus makeQualified(URI defaultUri, Path path) {
+    return new FileStatus(getLen(), isDir(), getReplication(),
+        getBlockSize(), getModificationTime(),
+        getAccessTime(),
+        getPermission(), getOwner(), getGroup(),
+        isSymlink() ? new Path(getSymlink()) : null,
+        (getFullPath(path)).makeQualified(
+            defaultUri, null)); // fully-qualify path
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
new file mode 100644
index 0000000..cc13f10
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Associates a block with the Datanodes that contain its replicas
+ * and other block metadata (E.g. the file offset associated with this
+ * block, whether it is corrupt, a location is cached in memory,
+ * security token, etc).
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class LocatedBlock {
+
+  private final ExtendedBlock b;
+  private long offset;  // offset of the first byte of the block in the file
+  private final DatanodeInfoWithStorage[] locs;
+  /** Cached storage ID for each replica */
+  private final String[] storageIDs;
+  /** Cached storage type for each replica, if reported. */
+  private final StorageType[] storageTypes;
+  // corrupt flag is true if all of the replicas of a block are corrupt.
+  // else false. If block has few corrupt replicas, they are filtered and
+  // their locations are not part of this object
+  private boolean corrupt;
+  private Token<BlockTokenIdentifier> blockToken = new Token<BlockTokenIdentifier>();
+  /**
+   * List of cached datanode locations
+   */
+  private DatanodeInfo[] cachedLocs;
+
+  // Used when there are no locations
+  private static final DatanodeInfoWithStorage[] EMPTY_LOCS =
+      new DatanodeInfoWithStorage[0];
+
+  public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) {
+    // By default, startOffset is unknown(-1) and corrupt is false.
+    this(b, locs, null, null, -1, false, EMPTY_LOCS);
+  }
+
+  public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs,
+                      String[] storageIDs, StorageType[] storageTypes) {
+    this(b, locs, storageIDs, storageTypes, -1, false, EMPTY_LOCS);
+  }
+
+  public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs,
+                      StorageType[] storageTypes, long startOffset,
+                      boolean corrupt, DatanodeInfo[] cachedLocs) {
+    this.b = b;
+    this.offset = startOffset;
+    this.corrupt = corrupt;
+    if (locs==null) {
+      this.locs = EMPTY_LOCS;
+    } else {
+      this.locs = new DatanodeInfoWithStorage[locs.length];
+      for(int i = 0; i < locs.length; i++) {
+        DatanodeInfo di = locs[i];
+        DatanodeInfoWithStorage storage = new DatanodeInfoWithStorage(di,
+            storageIDs != null ? storageIDs[i] : null,
+            storageTypes != null ? storageTypes[i] : null);
+        this.locs[i] = storage;
+      }
+    }
+    this.storageIDs = storageIDs;
+    this.storageTypes = storageTypes;
+
+    if (cachedLocs == null || cachedLocs.length == 0) {
+      this.cachedLocs = EMPTY_LOCS;
+    } else {
+      this.cachedLocs = cachedLocs;
+    }
+  }
+
+  public Token<BlockTokenIdentifier> getBlockToken() {
+    return blockToken;
+  }
+
+  public void setBlockToken(Token<BlockTokenIdentifier> token) {
+    this.blockToken = token;
+  }
+
+  public ExtendedBlock getBlock() {
+    return b;
+  }
+
+  /**
+   * Returns the locations associated with this block. The returned array is not
+   * expected to be modified. If it is, caller must immediately invoke
+   * {@link org.apache.hadoop.hdfs.protocol.LocatedBlock#updateCachedStorageInfo}
+   * to update the cached Storage ID/Type arrays.
+   */
+  public DatanodeInfo[] getLocations() {
+    return locs;
+  }
+
+  public StorageType[] getStorageTypes() {
+    return storageTypes;
+  }
+
+  public String[] getStorageIDs() {
+    return storageIDs;
+  }
+
+  /**
+   * Updates the cached StorageID and StorageType information. Must be
+   * called when the locations array is modified.
+   */
+  public void updateCachedStorageInfo() {
+    if (storageIDs != null) {
+      for(int i = 0; i < locs.length; i++) {
+        storageIDs[i] = locs[i].getStorageID();
+      }
+    }
+    if (storageTypes != null) {
+      for(int i = 0; i < locs.length; i++) {
+        storageTypes[i] = locs[i].getStorageType();
+      }
+    }
+  }
+
+  public long getStartOffset() {
+    return offset;
+  }
+
+  public long getBlockSize() {
+    return b.getNumBytes();
+  }
+
+  public void setStartOffset(long value) {
+    this.offset = value;
+  }
+
+  public void setCorrupt(boolean corrupt) {
+    this.corrupt = corrupt;
+  }
+
+  public boolean isCorrupt() {
+    return this.corrupt;
+  }
+
+  /**
+   * Add a the location of a cached replica of the block.
+   *
+   * @param loc of datanode with the cached replica
+   */
+  public void addCachedLoc(DatanodeInfo loc) {
+    List<DatanodeInfo> cachedList = Lists.newArrayList(cachedLocs);
+    if (cachedList.contains(loc)) {
+      return;
+    }
+    // Try to re-use a DatanodeInfo already in loc
+    for (DatanodeInfoWithStorage di : locs) {
+      if (loc.equals(di)) {
+        cachedList.add(di);
+        cachedLocs = cachedList.toArray(cachedLocs);
+        return;
+      }
+    }
+    // Not present in loc, add it and go
+    cachedList.add(loc);
+    cachedLocs = cachedList.toArray(cachedLocs);
+  }
+
+  /**
+   * @return Datanodes with a cached block replica
+   */
+  public DatanodeInfo[] getCachedLocations() {
+    return cachedLocs;
+  }
+
+  @Override
+  public String toString() {
+    return getClass().getSimpleName() + "{" + b
+        + "; getBlockSize()=" + getBlockSize()
+        + "; corrupt=" + corrupt
+        + "; offset=" + offset
+        + "; locs=" + Arrays.asList(locs)
+        + "}";
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
new file mode 100644
index 0000000..e4896977
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.util.List;
+import java.util.Collections;
+import java.util.Comparator;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+
+/**
+ * Collection of blocks with their locations and the file length.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class LocatedBlocks {
+  private final long fileLength;
+  private final List<LocatedBlock> blocks; // array of blocks with prioritized locations
+  private final boolean underConstruction;
+  private final LocatedBlock lastLocatedBlock;
+  private final boolean isLastBlockComplete;
+  private final FileEncryptionInfo fileEncryptionInfo;
+
+  public LocatedBlocks() {
+    fileLength = 0;
+    blocks = null;
+    underConstruction = false;
+    lastLocatedBlock = null;
+    isLastBlockComplete = false;
+    fileEncryptionInfo = null;
+  }
+
+  public LocatedBlocks(long flength, boolean isUnderConstuction,
+    List<LocatedBlock> blks, LocatedBlock lastBlock,
+    boolean isLastBlockCompleted, FileEncryptionInfo feInfo) {
+    fileLength = flength;
+    blocks = blks;
+    underConstruction = isUnderConstuction;
+    this.lastLocatedBlock = lastBlock;
+    this.isLastBlockComplete = isLastBlockCompleted;
+    this.fileEncryptionInfo = feInfo;
+  }
+
+  /**
+   * Get located blocks.
+   */
+  public List<LocatedBlock> getLocatedBlocks() {
+    return blocks;
+  }
+
+  /** Get the last located block. */
+  public LocatedBlock getLastLocatedBlock() {
+    return lastLocatedBlock;
+  }
+
+  /** Is the last block completed? */
+  public boolean isLastBlockComplete() {
+    return isLastBlockComplete;
+  }
+
+  /**
+   * Get located block.
+   */
+  public LocatedBlock get(int index) {
+    return blocks.get(index);
+  }
+
+  /**
+   * Get number of located blocks.
+   */
+  public int locatedBlockCount() {
+    return blocks == null ? 0 : blocks.size();
+  }
+
+  /**
+   *
+   */
+  public long getFileLength() {
+    return this.fileLength;
+  }
+
+  /**
+   * Return true if file was under construction when this LocatedBlocks was
+   * constructed, false otherwise.
+   */
+  public boolean isUnderConstruction() {
+    return underConstruction;
+  }
+
+  /**
+   * @return the FileEncryptionInfo for the LocatedBlocks
+   */
+  public FileEncryptionInfo getFileEncryptionInfo() {
+    return fileEncryptionInfo;
+  }
+
+  /**
+   * Find block containing specified offset.
+   *
+   * @return block if found, or null otherwise.
+   */
+  public int findBlock(long offset) {
+    // create fake block of size 0 as a key
+    LocatedBlock key = new LocatedBlock(
+        new ExtendedBlock(), new DatanodeInfo[0]);
+    key.setStartOffset(offset);
+    key.getBlock().setNumBytes(1);
+    Comparator<LocatedBlock> comp =
+      new Comparator<LocatedBlock>() {
+        // Returns 0 iff a is inside b or b is inside a
+        @Override
+        public int compare(LocatedBlock a, LocatedBlock b) {
+          long aBeg = a.getStartOffset();
+          long bBeg = b.getStartOffset();
+          long aEnd = aBeg + a.getBlockSize();
+          long bEnd = bBeg + b.getBlockSize();
+          if(aBeg <= bBeg && bEnd <= aEnd
+              || bBeg <= aBeg && aEnd <= bEnd)
+            return 0; // one of the blocks is inside the other
+          if(aBeg < bBeg)
+            return -1; // a's left bound is to the left of the b's
+          return 1;
+        }
+      };
+    return Collections.binarySearch(blocks, key, comp);
+  }
+
+  public void insertRange(int blockIdx, List<LocatedBlock> newBlocks) {
+    int oldIdx = blockIdx;
+    int insStart = 0, insEnd = 0;
+    for(int newIdx = 0; newIdx < newBlocks.size() && oldIdx < blocks.size();
+                                                        newIdx++) {
+      long newOff = newBlocks.get(newIdx).getStartOffset();
+      long oldOff = blocks.get(oldIdx).getStartOffset();
+      if(newOff < oldOff) {
+        insEnd++;
+      } else if(newOff == oldOff) {
+        // replace old cached block by the new one
+        blocks.set(oldIdx, newBlocks.get(newIdx));
+        if(insStart < insEnd) { // insert new blocks
+          blocks.addAll(oldIdx, newBlocks.subList(insStart, insEnd));
+          oldIdx += insEnd - insStart;
+        }
+        insStart = insEnd = newIdx+1;
+        oldIdx++;
+      } else {  // newOff > oldOff
+        assert false : "List of LocatedBlock must be sorted by startOffset";
+      }
+    }
+    insEnd = newBlocks.size();
+    if(insStart < insEnd) { // insert new blocks
+      blocks.addAll(oldIdx, newBlocks.subList(insStart, insEnd));
+    }
+  }
+
+  public static int getInsertIndex(int binSearchResult) {
+    return binSearchResult >= 0 ? binSearchResult : -(binSearchResult+1);
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder b = new StringBuilder(getClass().getSimpleName());
+    b.append("{")
+     .append("\n  fileLength=").append(fileLength)
+     .append("\n  underConstruction=").append(underConstruction)
+     .append("\n  blocks=").append(blocks)
+     .append("\n  lastLocatedBlock=").append(lastLocatedBlock)
+     .append("\n  isLastBlockComplete=").append(isLastBlockComplete)
+     .append("}");
+    return b.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
new file mode 100644
index 0000000..3f2c9ca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security.token.block;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+@InterfaceAudience.Private
+public class BlockTokenIdentifier extends TokenIdentifier {
+  static final Text KIND_NAME = new Text("HDFS_BLOCK_TOKEN");
+
+  public enum AccessMode {
+    READ, WRITE, COPY, REPLACE
+  }
+
+  private long expiryDate;
+  private int keyId;
+  private String userId;
+  private String blockPoolId;
+  private long blockId;
+  private final EnumSet<AccessMode> modes;
+
+  private byte [] cache;
+
+  public BlockTokenIdentifier() {
+    this(null, null, 0, EnumSet.noneOf(AccessMode.class));
+  }
+
+  public BlockTokenIdentifier(String userId, String bpid, long blockId,
+      EnumSet<AccessMode> modes) {
+    this.cache = null;
+    this.userId = userId;
+    this.blockPoolId = bpid;
+    this.blockId = blockId;
+    this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
+  }
+
+  @Override
+  public Text getKind() {
+    return KIND_NAME;
+  }
+
+  @Override
+  public UserGroupInformation getUser() {
+    if (userId == null || "".equals(userId)) {
+      String user = blockPoolId + ":" + Long.toString(blockId);
+      return UserGroupInformation.createRemoteUser(user);
+    }
+    return UserGroupInformation.createRemoteUser(userId);
+  }
+
+  public long getExpiryDate() {
+    return expiryDate;
+  }
+
+  public void setExpiryDate(long expiryDate) {
+    this.cache = null;
+    this.expiryDate = expiryDate;
+  }
+
+  public int getKeyId() {
+    return this.keyId;
+  }
+
+  public void setKeyId(int keyId) {
+    this.cache = null;
+    this.keyId = keyId;
+  }
+
+  public String getUserId() {
+    return userId;
+  }
+
+  public String getBlockPoolId() {
+    return blockPoolId;
+  }
+
+  public long getBlockId() {
+    return blockId;
+  }
+
+  public EnumSet<AccessMode> getAccessModes() {
+    return modes;
+  }
+
+  @Override
+  public String toString() {
+    return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+        + ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId()
+        + ", blockPoolId=" + this.getBlockPoolId()
+        + ", blockId=" + this.getBlockId() + ", access modes="
+        + this.getAccessModes() + ")";
+  }
+
+  static boolean isEqual(Object a, Object b) {
+    return a == null ? b == null : a.equals(b);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (obj instanceof BlockTokenIdentifier) {
+      BlockTokenIdentifier that = (BlockTokenIdentifier) obj;
+      return this.expiryDate == that.expiryDate && this.keyId == that.keyId
+          && isEqual(this.userId, that.userId)
+          && isEqual(this.blockPoolId, that.blockPoolId)
+          && this.blockId == that.blockId
+          && isEqual(this.modes, that.modes);
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return (int) expiryDate ^ keyId ^ (int) blockId ^ modes.hashCode()
+        ^ (userId == null ? 0 : userId.hashCode())
+        ^ (blockPoolId == null ? 0 : blockPoolId.hashCode());
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    this.cache = null;
+    expiryDate = WritableUtils.readVLong(in);
+    keyId = WritableUtils.readVInt(in);
+    userId = WritableUtils.readString(in);
+    blockPoolId = WritableUtils.readString(in);
+    blockId = WritableUtils.readVLong(in);
+    int length = WritableUtils.readVIntInRange(in, 0,
+        AccessMode.class.getEnumConstants().length);
+    for (int i = 0; i < length; i++) {
+      modes.add(WritableUtils.readEnum(in, AccessMode.class));
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    WritableUtils.writeVLong(out, expiryDate);
+    WritableUtils.writeVInt(out, keyId);
+    WritableUtils.writeString(out, userId);
+    WritableUtils.writeString(out, blockPoolId);
+    WritableUtils.writeVLong(out, blockId);
+    WritableUtils.writeVInt(out, modes.size());
+    for (AccessMode aMode : modes) {
+      WritableUtils.writeEnum(out, aMode);
+    }
+  }
+
+  @Override
+  public byte[] getBytes() {
+    if(cache == null) cache = super.getBytes();
+
+    return cache;
+  }
+
+  @InterfaceAudience.Private
+  public static class Renewer extends Token.TrivialRenewer {
+    @Override
+    protected Text getKind() {
+      return KIND_NAME;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
new file mode 100644
index 0000000..e8d7439
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security.token.delegation;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/**
+ * A delegation token identifier that is specific to HDFS.
+ */
+@InterfaceAudience.Private
+public class DelegationTokenIdentifier
+    extends AbstractDelegationTokenIdentifier {
+  public static final Text HDFS_DELEGATION_KIND = new Text("HDFS_DELEGATION_TOKEN");
+
+  /**
+   * Create an empty delegation token identifier for reading into.
+   */
+  public DelegationTokenIdentifier() {
+  }
+
+  /**
+   * Create a new delegation token identifier
+   * @param owner the effective username of the token owner
+   * @param renewer the username of the renewer
+   * @param realUser the real username of the token owner
+   */
+  public DelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
+    super(owner, renewer, realUser);
+  }
+
+  @Override
+  public Text getKind() {
+    return HDFS_DELEGATION_KIND;
+  }
+
+  @Override
+  public String toString() {
+    return getKind() + " token " + getSequenceNumber()
+        + " for " + getUser().getShortUserName();
+  }
+
+  /** @return a string representation of the token */
+  public static String stringifyToken(final Token<?> token) throws IOException {
+    DelegationTokenIdentifier ident = new DelegationTokenIdentifier();
+    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+    DataInputStream in = new DataInputStream(buf);
+    ident.readFields(in);
+
+    if (token.getService().getLength() > 0) {
+      return ident + " on " + token.getService();
+    } else {
+      return ident.toString();
+    }
+  }
+
+  public static class WebHdfsDelegationTokenIdentifier
+      extends DelegationTokenIdentifier {
+    public WebHdfsDelegationTokenIdentifier() {
+      super();
+    }
+    @Override
+    public Text getKind() {
+      return WebHdfsConstants.WEBHDFS_TOKEN_KIND;
+    }
+  }
+
+  public static class SWebHdfsDelegationTokenIdentifier
+      extends WebHdfsDelegationTokenIdentifier {
+    public SWebHdfsDelegationTokenIdentifier() {
+      super();
+    }
+    @Override
+    public Text getKind() {
+      return WebHdfsConstants.SWEBHDFS_TOKEN_KIND;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
new file mode 100644
index 0000000..25f3cfd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.io.Text;
+
+@InterfaceAudience.Private
+public class WebHdfsConstants {
+  /** Delegation token kind */
+  public static final Text WEBHDFS_TOKEN_KIND = new Text("WEBHDFS delegation");
+  public static final Text SWEBHDFS_TOKEN_KIND = new Text("SWEBHDFS delegation");
+
+  enum PathType {
+    FILE, DIRECTORY, SYMLINK;
+
+    static PathType valueOf(HdfsFileStatus status) {
+      return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1ef60be..7da514a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -132,6 +132,8 @@ Release 2.8.0 - UNRELEASED
 
     HDFS-8152. Refactoring of lazy persist storage cases. (Arpit Agarwal)
 
+    HDFS-8169. Move LocatedBlocks and related classes to hdfs-client. (wheat9)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d3a4d51/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 846231a..598516c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -1235,21 +1235,6 @@ public class DFSUtil {
       }
     }
   }
-  
-  /** Return used as percentage of capacity */
-  public static float getPercentUsed(long used, long capacity) {
-    return capacity <= 0 ? 100 : (used * 100.0f)/capacity; 
-  }
-  
-  /** Return remaining as percentage of capacity */
-  public static float getPercentRemaining(long remaining, long capacity) {
-    return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; 
-  }
-
-  /** Convert percentage to a string. */
-  public static String percent2String(double percentage) {
-    return StringUtils.format("%.2f%%", percentage);
-  }
 
   /**
    * Round bytes to GiB (gibibyte)


Mime
View raw message