Author: szetszwo Date: Wed Sep 9 20:46:41 2009 New Revision: 813111 URL: http://svn.apache.org/viewvc?rev=813111&view=rev Log: HDFS-603. Add a new interface, Replica, which is going to replace the use of Block in datanode. Added: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/Replica.java Modified: hadoop/hdfs/branches/HDFS-265/CHANGES.txt hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaWaitingToBeRecovered.java hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Modified: hadoop/hdfs/branches/HDFS-265/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/CHANGES.txt?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/CHANGES.txt (original) +++ hadoop/hdfs/branches/HDFS-265/CHANGES.txt Wed Sep 9 20:46:41 2009 @@ -33,6 +33,9 @@ HDFS-543. Break FSDatasetInterface#writToBlock() into writeToRemporary, writeToRBW, ad append. (hairong) + HDFS-603. Add a new interface, Replica, which is going to replace the use + of Block in datanode. (szetszwo) + BUG FIXES HDFS-547. TestHDFSFileSystemContract#testOutputStreamClosedTwice Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Wed Sep 9 20:46:41 2009 @@ -1832,14 +1832,8 @@ } } - /** - * Get reference to the replica meta info in the replicasMap. - * To be called from methods that - * are synchronized on {@link FSDataset} - * @param blockId - * @return replica's meta information from the replicas map - */ - ReplicaInfo getBlock(long blockId) { + @Override + public ReplicaInfo getReplica(long blockId) { assert(Thread.holdsLock(this)); return volumeMap.get(blockId); } Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java Wed Sep 9 20:46:41 2009 @@ -95,6 +95,14 @@ public long getLength(Block b) throws IOException; /** + * Get reference to the replica meta info in the replicasMap. + * To be called from methods that are synchronized on {@link FSDataset} + * @param blockId + * @return replica from the replicas map + */ + public Replica getReplica(long blockId); + + /** * @return the generation stamp stored with the block. */ public Block getStoredBlock(long blkid) throws IOException; Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java Wed Sep 9 20:46:41 2009 @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; -import java.io.IOException; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; @@ -68,11 +67,16 @@ detached = true; } - @Override // ReplicaInfo - long getVisibleLen() throws IOException { + @Override + public long getVisibleLength() { return getNumBytes(); // all bytes are visible } - + + @Override + public long getBytesOnDisk() { + return getNumBytes(); + } + @Override // Object public boolean equals(Object o) { return super.equals(o); @@ -82,4 +86,10 @@ public int hashCode() { return super.hashCode(); } + + @Override + public String toString() { + return super.toString() + + "\n detached=" + detached; + } } Added: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/Replica.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/Replica.java?rev=813111&view=auto ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/Replica.java (added) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/Replica.java Wed Sep 9 20:46:41 2009 @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; + +/** + * This represents block replicas which stored in DataNode. + */ +public interface Replica { + /** get block ID */ + public long getBlockId(); + + /** get generation stamp */ + public long getGenerationStamp(); + + /** + * Get the replica state + * @return the replica state + */ + public ReplicaState getState(); + + /** + * Get the number of bytes received + * @return the number of bytes that have been received + */ + public long getNumBytes(); + + /** + * Get the number of bytes that have written to disk + * @return the number of bytes that have written to disk + */ + public long getBytesOnDisk(); + + /** + * Get the number of bytes that are visible to readers + * @return the number of bytes that are visible to readers + */ + public long getVisibleLength(); +} Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaBeingWritten.java Wed Sep 9 20:46:41 2009 @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; -import java.io.IOException; + import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.FSDataset.FSVolume; @@ -66,8 +66,8 @@ super( blockId, len, genStamp, vol, dir, writer); } - @Override //ReplicaInfo - long getVisibleLen() throws IOException { + @Override + public long getVisibleLength() { return getBytesAcked(); // all acked bytes are visible } Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java Wed Sep 9 20:46:41 2009 @@ -85,10 +85,9 @@ this.writer = writer; } - @Override //ReplicaInfo - long getVisibleLen() throws IOException { - // no bytes are visible - throw new IOException("No bytes are visible for temporary replicas"); + @Override + public long getVisibleLength() { + return -1; } @Override //ReplicaInfo @@ -172,5 +171,12 @@ IOUtils.closeStream(crcOut); throw e; } - } + } + + @Override + public String toString() { + return super.toString() + + "\n bytesAcked=" + bytesAcked + + "\n bytesOnDisk=" + bytesOnDisk; + } } Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java Wed Sep 9 20:46:41 2009 @@ -24,13 +24,7 @@ /** * This defines the interface of a replica in Pipeline that's being written to */ -interface ReplicaInPipelineInterface { - /** - * Get the number of bytes received - * @return the number of bytes that have been received - */ - long getNumBytes(); - +interface ReplicaInPipelineInterface extends Replica { /** * Set the number of bytes received * @param bytesReceived number of bytes received @@ -50,12 +44,6 @@ void setBytesAcked(long bytesAcked); /** - * Get the number of bytes that have written to disk - * @return the number of bytes that have written to disk - */ - long getBytesOnDisk(); - - /** * Set the number of bytes on disk * @param bytesOnDisk number of bytes on disk */ Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java Wed Sep 9 20:46:41 2009 @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil.HardLink; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.FSDataset.FSVolume; import org.apache.hadoop.io.IOUtils; @@ -33,7 +32,7 @@ * This class is used by datanodes to maintain meta data of its replicas. * It provides a general interface for meta information of a replica. */ -abstract public class ReplicaInfo extends Block { +abstract public class ReplicaInfo extends Block implements Replica { private FSVolume volume; // volume where the replica belongs private File dir; // directory where block & meta files belong @@ -129,13 +128,6 @@ this.dir = dir; } - - /** - * Get the replica state - * @return the replica state - */ - abstract public ReplicaState getState(); - /** * check if this replica has already detached. * @return true if the replica has already detached or no need to detach; @@ -222,12 +214,6 @@ } /** - * Get the number of bytes that are visible to readers - * @return the number of bytes that are visible to readers - */ - abstract long getVisibleLen() throws IOException; - - /** * Set this replica's generation stamp to be a newer one * @param newGS new generation stamp * @throws IOException is the new generation stamp is not greater than the current one @@ -243,7 +229,13 @@ @Override //Object public String toString() { - return getClass().getSimpleName() + " " + super.toString() + - "(volume=" + volume + ", file=" + getBlockFile() + ")"; + return getClass().getSimpleName() + + ", " + super.toString() + + ", " + getState() + + "\n getNumBytes() = " + getNumBytes() + + "\n getBytesOnDisk() = " + getBytesOnDisk() + + "\n getVisibleLength()= " + getVisibleLength() + + "\n getVolume() = " + getVolume() + + "\n getBlockFile() = " + getBlockFile(); } } Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaUnderRecovery.java Wed Sep 9 20:46:41 2009 @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; -import java.io.IOException; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.FSDataset.FSVolume; @@ -101,8 +100,13 @@ } @Override - long getVisibleLen() throws IOException { - return original.getVisibleLen(); + public long getVisibleLength() { + return original.getVisibleLength(); + } + + @Override + public long getBytesOnDisk() { + return original.getBytesOnDisk(); } @Override //org.apache.hadoop.hdfs.protocol.Block @@ -144,4 +148,11 @@ public int hashCode() { return super.hashCode(); } + + @Override + public String toString() { + return super.toString() + + "\n recoveryId=" + recoveryId + + "\n original=" + original; + } } Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaWaitingToBeRecovered.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaWaitingToBeRecovered.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaWaitingToBeRecovered.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaWaitingToBeRecovered.java Wed Sep 9 20:46:41 2009 @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; -import java.io.IOException; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; @@ -75,10 +74,15 @@ } @Override //ReplicaInfo - long getVisibleLen() throws IOException { + public long getVisibleLength() { return -1; //no bytes are visible } + @Override + public long getBytesOnDisk() { + return getNumBytes(); + } + @Override // Object public boolean equals(Object o) { return super.equals(o); @@ -88,4 +92,10 @@ public int hashCode() { return super.hashCode(); } + + @Override + public String toString() { + return super.toString() + + "\n detached=" + detached; + } } Modified: hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Wed Sep 9 20:46:41 2009 @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.metrics.util.MBeanUtil; import org.apache.hadoop.util.DataChecksum; @@ -107,7 +108,7 @@ } } - synchronized long getGenerationStamp() { + synchronized public long getGenerationStamp() { return theBlock.getGenerationStamp(); } @@ -210,6 +211,21 @@ } @Override + synchronized public long getBlockId() { + return theBlock.getBlockId(); + } + + @Override + synchronized public long getVisibleLength() { + return getBytesAcked(); + } + + @Override + public ReplicaState getState() { + return null; + } + + @Override synchronized public long getBytesAcked() { if (finalized) { return theBlock.getNumBytes(); @@ -379,6 +395,11 @@ return binfo.getNumBytes(); } + @Override + public Replica getReplica(long blockId) { + return blockMap.get(new Block(blockId)); + } + /** {@inheritDoc} */ public Block getStoredBlock(long blkid) throws IOException { Block b = new Block(blkid); Modified: hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java?rev=813111&r1=813110&r2=813111&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java (original) +++ hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Wed Sep 9 20:46:41 2009 @@ -317,23 +317,21 @@ } private void verifyAddition(long blockId, long genStamp, long size) { - Block memBlock = fds.getBlock(blockId); - assertNotNull(memBlock); - ReplicaInfo blockInfo; + final ReplicaInfo replicainfo; synchronized(fds) { - blockInfo = fds.volumeMap.get(memBlock); + replicainfo = fds.getReplica(blockId); } - assertNotNull(blockInfo); + assertNotNull(replicainfo); // Added block has the same file as the one created by the test File file = new File(getBlockFile(blockId)); - assertEquals(file.getName(), blockInfo.getBlockFile().getName()); + assertEquals(file.getName(), replicainfo.getBlockFile().getName()); // Generation stamp is same as that of created file - assertEquals(genStamp, memBlock.getGenerationStamp()); + assertEquals(genStamp, replicainfo.getGenerationStamp()); // File size matches - assertEquals(size, memBlock.getNumBytes()); + assertEquals(size, replicainfo.getNumBytes()); } private void verifyDeletion(long blockId) { @@ -344,9 +342,9 @@ } private void verifyGenStamp(long blockId, long genStamp) { - Block memBlock; + final Replica memBlock; synchronized(fds) { - memBlock = fds.getBlock(blockId); + memBlock = fds.getReplica(blockId); } assertNotNull(memBlock); assertEquals(genStamp, memBlock.getGenerationStamp());