Author: dhruba
Date: Fri May 9 14:32:26 2008
New Revision: 654941
URL: http://svn.apache.org/viewvc?rev=654941&view=rev
Log:
HADOOP-3329. DatanodeDescriptor objects should not be stored in the
fsimage. (dhruba)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri May 9 14:32:26 2008
@@ -44,6 +44,9 @@
HADOOP-3226. Run combiners multiple times over map outputs as they
are merged in both the map and the reduce tasks. (cdouglas via omalley)
+ HADOOP-3329. DatanodeDescriptor objects should not be stored in the
+ fsimage. (dhruba)
+
NEW FEATURES
HADOOP-3074. Provides a UrlStreamHandler for DFS and other FS,
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java Fri May 9 14:32:26
2008
@@ -374,23 +374,6 @@
}
/** Serialization for FSEditLog */
- //TODO: remove this method in HADOOP-3329
- void write2FSEditLog(DataOutput out) throws IOException {
- UTF8.writeString(out, name);
- UTF8.writeString(out, storageID);
- out.writeShort(infoPort);
- out.writeLong(capacity);
- out.writeLong(dfsUsed);
- out.writeLong(remaining);
- out.writeLong(lastUpdate);
- out.writeInt(xceiverCount);
- Text.writeString(out, location);
- Text.writeString(out, hostName == null? "": hostName);
- WritableUtils.writeEnum(out, getAdminState());
- }
-
- /** Serialization for FSEditLog */
- //TODO: remove this method in HADOOP-3329
void readFieldsFromFSEditLog(DataInput in) throws IOException {
this.name = UTF8.readString(in);
this.storageID = UTF8.readString(in);
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSConstants.java Fri May 9 14:32:26
2008
@@ -190,7 +190,7 @@
// Version is reflected in the data storage file.
// Versions are negative.
// Decrement LAYOUT_VERSION to define a new version.
- public static final int LAYOUT_VERSION = -13;
+ public static final int LAYOUT_VERSION = -14;
// Current version:
- // Fix bug introduced by OPEN, CLOSE and GENSTAMP transactions for supporting appends
+ // Remove storing locations of last block of a file in fsimage
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDirectory.java Fri May 9 14:32:26
2008
@@ -309,9 +309,6 @@
fileNode.removeBlock(block);
namesystem.blocksMap.removeINode(block);
- // Remove the block locations for the last block.
- fileNode.setLastBlockLocations(new DatanodeDescriptor[0]);
-
// write modified block locations to log
fsImage.getEditLog().logOpenFile(path, fileNode);
NameNode.stateChangeLog.debug("DIR* FSDirectory.addFile: "
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java Fri May 9 14:32:26 2008
@@ -384,7 +384,6 @@
INode old = null;
String clientName = null;
String clientMachine = null;
- DatanodeDescriptor lastLocations[] = null;
String path = null;
int numOpAdd = 0, numOpClose = 0, numOpDelete = 0,
numOpRename = 0, numOpSetRepl = 0, numOpMkDir = 0,
@@ -470,15 +469,15 @@
}
// clientname, clientMachine and block locations of last block.
- lastLocations = null;
if (opcode == OP_ADD && logVersion <= -12) {
clientName = FSImage.readString(in);
clientMachine = FSImage.readString(in);
- lastLocations = readDatanodeDescriptorArray(in);
+ if (-13 <= logVersion) {
+ readDatanodeDescriptorArray(in);
+ }
} else {
clientName = "";
clientMachine = "";
- lastLocations = EMPTY_ARRAY_DN_DESCRIPTORS;
}
// The open lease transaction re-creates a file if necessary.
@@ -512,8 +511,7 @@
node.getPermissionStatus(),
clientName,
clientMachine,
- null,
- lastLocations);
+ null);
fsDir.replaceNode(path, node, cons);
fsNamesys.leaseManager.addLease(path, clientName);
} else if (opcode == OP_CLOSE) {
@@ -800,7 +798,6 @@
*/
void logOpenFile(String path, INodeFileUnderConstruction newNode)
throws IOException {
- final DatanodeDescriptor[] locations = newNode.getLastBlockLocations();
UTF8 nameReplicationPair[] = new UTF8[] {
new UTF8(path),
@@ -812,13 +809,7 @@
new ArrayWritable(Block.class, newNode.getBlocks()),
newNode.getPermissionStatus(),
new UTF8(newNode.getClientName()),
- new UTF8(newNode.getClientMachine()),
- new Writable() {
- public void readFields(DataInput in) {}
- public void write(DataOutput out) throws IOException {
- writeDatanodeDescriptorArray(out, locations);
- }
- });
+ new UTF8(newNode.getClientMachine()));
}
/**
@@ -1013,19 +1004,6 @@
}
/** This method is defined for compatibility reason. */
- //TODO: remove this class in HADOOP-3329
- static private void writeDatanodeDescriptorArray(DataOutput out,
- DatanodeDescriptor[] locations) throws IOException {
- out.writeInt(locations.length); // write values
- for (int i = 0; i < locations.length; i++) {
- locations[i].write2FSEditLog(out);
- }
- }
-
- /** This method is defined for compatibility reason. */
- private static final DatanodeDescriptor[] EMPTY_ARRAY_DN_DESCRIPTORS
- = new DatanodeDescriptor[0];
- //TODO: remove this class in HADOOP-3329
static private DatanodeDescriptor[] readDatanodeDescriptorArray(DataInput in
) throws IOException {
DatanodeDescriptor[] locations = new DatanodeDescriptor[in.readInt()];
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Fri May 9 14:32:26 2008
@@ -1035,6 +1035,7 @@
String clientName = readString(in);
String clientMachine = readString(in);
+ // These locations are not used at all
int numLocs = in.readInt();
DatanodeDescriptor[] locations = new DatanodeDescriptor[numLocs];
for (int i = 0; i < numLocs; i++) {
@@ -1050,9 +1051,7 @@
perm,
clientName,
clientMachine,
- null,
- locations);
-
+ null);
}
// Helper function that writes an INodeUnderConstruction
@@ -1075,11 +1074,7 @@
writeString(cons.getClientName(), out);
writeString(cons.getClientMachine(), out);
- int numLocs = cons.getLastBlockLocations().length;
- out.writeInt(numLocs);
- for (int i = 0; i < numLocs; i++) {
- cons.getLastBlockLocations()[i].write(out);
- }
+ out.writeInt(0); // do not store locations of last block
}
/**
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri May 9 14:32:26
2008
@@ -1015,7 +1015,6 @@
// allocate new block record block locations in INode.
newBlock = allocateBlock(src, pendingFile);
- pendingFile.setLastBlockLocations(targets);
}
// Create next block
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java?rev=654941&r1=654940&r2=654941&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java Fri May 9 14:32:26 2008
@@ -781,7 +781,6 @@
protected StringBytesWritable clientName; // lease holder
protected StringBytesWritable clientMachine;
protected DatanodeDescriptor clientNode; // if client is a cluster node too.
- protected DatanodeDescriptor[] targets; // locations for last block
INodeFileUnderConstruction() {
clientName = null;
@@ -802,7 +801,6 @@
this.clientName = new StringBytesWritable(clientName);
this.clientMachine = new StringBytesWritable(clientMachine);
this.clientNode = clientNode;
- this.targets = new DatanodeDescriptor[0];
}
INodeFileUnderConstruction(byte[] name,
@@ -813,8 +811,7 @@
PermissionStatus perm,
String clientName,
String clientMachine,
- DatanodeDescriptor clientNode,
- DatanodeDescriptor[] targets)
+ DatanodeDescriptor clientNode)
throws IOException {
super(perm, blocks, blockReplication, modificationTime,
preferredBlockSize);
@@ -822,7 +819,6 @@
this.clientName = new StringBytesWritable(clientName);
this.clientMachine = new StringBytesWritable(clientMachine);
this.clientNode = clientNode;
- this.targets = targets;
}
String getClientName() throws IOException {
@@ -837,14 +833,6 @@
return clientNode;
}
- void setLastBlockLocations(DatanodeDescriptor[] targets) {
- this.targets = targets;
- }
-
- DatanodeDescriptor[] getLastBlockLocations() {
- return this.targets;
- }
-
/**
* Is this inode being constructed?
*/
|