Author: hairong
Date: Sat Jan 23 01:35:01 2010
New Revision: 902330
URL: http://svn.apache.org/viewvc?rev=902330&view=rev
Log:
HDFS-800. The last block of a file under construction may change to the COMPLETE state in
response to getAdditionalBlock or completeFileInternal. Contributed by Hairong Kuang.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=902330&r1=902329&r2=902330&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Sat Jan 23 01:35:01 2010
@@ -38,6 +38,10 @@
HDFS-902 Move contrib/raid to MapReduce. (Eli Collins via omalley)
+ HDFS-800. The last block of a file under construction may change to the
+ COMPLETE state in response to getAdditionalBlock or completeFileInternal.
+ (hairong)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=902330&r1=902329&r2=902330&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Sat
Jan 23 01:35:01 2010
@@ -245,26 +245,23 @@
}
/**
- * Commit the last block of the file.
+ * Commit a block of a file
*
* @param fileINode file inode
+ * @param block block to be committed
* @param commitBlock - contains client reported block length and generation
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
- void commitLastBlock(INodeFileUnderConstruction fileINode,
+ private void commitBlock(INodeFileUnderConstruction fileINode,
+ BlockInfoUnderConstruction block,
Block commitBlock) throws IOException {
- if(commitBlock == null)
- return; // not committing, this is a block allocation retry
- BlockInfo lastBlock = fileINode.getLastBlock();
- if(lastBlock == null)
- return; // no blocks in file yet
- if(lastBlock.isComplete())
- return; // already completed (e.g. by syncBlock)
- assert lastBlock.getNumBytes() <= commitBlock.getNumBytes() :
+ if (block.getBlockUCState() == BlockUCState.COMMITTED)
+ return;
+ assert block.getNumBytes() <= commitBlock.getNumBytes() :
"commitBlock length is less than the stored one "
- + commitBlock.getNumBytes() + " vs. " + lastBlock.getNumBytes();
- ((BlockInfoUnderConstruction)lastBlock).commitBlock(commitBlock);
+ + commitBlock.getNumBytes() + " vs. " + block.getNumBytes();
+ block.commitBlock(commitBlock);
// Adjust disk space consumption if required
long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
@@ -281,6 +278,32 @@
}
}
}
+
+ /**
+ * Commit the last block of the file and mark it as complete if it has
+ * meets the minimum replication requirement
+ *
+ * @param fileINode file inode
+ * @param commitBlock - contains client reported block length and generation
+ * @throws IOException if the block does not have at least a minimal number
+ * of replicas reported from data-nodes.
+ */
+ void commitOrCompleteLastBlock(INodeFileUnderConstruction fileINode,
+ Block commitBlock) throws IOException {
+
+ if(commitBlock == null)
+ return; // not committing, this is a block allocation retry
+ BlockInfo lastBlock = fileINode.getLastBlock();
+ if(lastBlock == null)
+ return; // no blocks in file yet
+ if(lastBlock.isComplete())
+ return; // already completed (e.g. by syncBlock)
+
+ commitBlock(fileINode, (BlockInfoUnderConstruction)lastBlock, commitBlock);
+
+ if(countNodes(lastBlock).liveReplicas() >= minReplication)
+ completeBlock(fileINode,fileINode.numBlocks()-1);
+ }
/**
* Convert a specified block of the file to a complete block.
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=902330&r1=902329&r2=902330&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat
Jan 23 01:35:01 2010
@@ -1326,8 +1326,8 @@
INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
- // commit the last block
- blockManager.commitLastBlock(pendingFile, previous);
+ // commit the last block and complete it if it has minimum replicas
+ blockManager.commitOrCompleteLastBlock(pendingFile, previous);
//
// If we fail this, bad things happen!
@@ -1362,9 +1362,6 @@
throw new NotReplicatedYetException("Not replicated yet:" + src);
}
- // complete the penultimate block
- blockManager.completeBlock(pendingFile, pendingFile.numBlocks()-2);
-
// allocate new block record block locations in INode.
newBlock = allocateBlock(src, pathINodes, targets);
@@ -1480,8 +1477,8 @@
return CompleteFileStatus.OPERATION_FAILED;
}
- // commit the last block
- blockManager.commitLastBlock(pendingFile, last);
+ // commit the last block and complete it if it has minimum replicas
+ blockManager.commitOrCompleteLastBlock(pendingFile, last);
if (!checkFileProgress(pendingFile, true)) {
return CompleteFileStatus.STILL_WAITING;
@@ -1541,7 +1538,7 @@
// check all blocks of the file.
//
for (BlockInfo block: v.getBlocks()) {
- if (!blockManager.checkMinReplication(block)) {
+ if (!block.isComplete()) {
LOG.info("BLOCK* NameSystem.checkFileProgress: "
+ "block " + block + " has not reached minimal replication "
+ blockManager.minReplication);
@@ -1553,7 +1550,7 @@
// check the penultimate block of this file
//
BlockInfo b = v.getPenultimateBlock();
- if (b != null && !blockManager.checkMinReplication(b)) {
+ if (b != null && !b.isComplete()) {
LOG.info("BLOCK* NameSystem.checkFileProgress: "
+ "block " + b + " has not reached minimal replication "
+ blockManager.minReplication);
@@ -2020,16 +2017,11 @@
INodeFileUnderConstruction pendingFile) throws IOException {
leaseManager.removeLease(pendingFile.getClientName(), src);
- // complete the penultimate block
- blockManager.completeBlock(pendingFile, pendingFile.numBlocks()-2);
-
// The file is no longer pending.
// Create permanent INode, update blocks
INodeFile newFile = pendingFile.convertToInodeFile();
dir.replaceNode(src, pendingFile, newFile);
- // complete last block of the file
- blockManager.completeBlock(newFile, newFile.numBlocks()-1);
// close file and persist block allocations for this file
dir.closeFile(src, newFile);
@@ -2111,8 +2103,8 @@
return;
}
- // commit the last block
- blockManager.commitLastBlock(pendingFile, storedBlock);
+ // commit the last block and complete it if it has minimum replicas
+ blockManager.commitOrCompleteLastBlock(pendingFile, storedBlock);
//remove lease, close file
finalizeINodeFileUnderConstruction(src, pendingFile);
|