hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1099641 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/DFSClient.java src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java src/java/org/apache/hadoop/hdfs/LeaseRenewer.java
Date Wed, 04 May 2011 23:52:03 GMT
Author: szetszwo
Date: Wed May  4 23:52:03 2011
New Revision: 1099641

URL: http://svn.apache.org/viewvc?rev=1099641&view=rev
Log:
HDFS-1890. Improve the name, class and value type of the map LeaseRenewer.pendingCreates.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1099641&r1=1099640&r2=1099641&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Wed May  4 23:52:03 2011
@@ -496,6 +496,9 @@ Trunk (unreleased changes)
 
     HDFS-1889. incorrect path in start/stop dfs script. (John George via eli)
 
+    HDFS-1890. Improve the name, class and value type of the map
+    LeaseRenewer.pendingCreates.  (szetszwo)
+
 Release 0.22.0 - Unreleased
 
   NEW FEATURES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1099641&r1=1099640&r2=1099641&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java Wed May  4 23:52:03 2011
@@ -629,7 +629,7 @@ public class DFSClient implements FSCons
     if(LOG.isDebugEnabled()) {
       LOG.debug(src + ": masked=" + masked);
     }
-    OutputStream result = new DFSOutputStream(this, src, masked,
+    final DFSOutputStream result = new DFSOutputStream(this, src, masked,
         flag, createParent, replication, blockSize, progress, buffersize,
         conf.getInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 
                     DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT));
@@ -640,7 +640,7 @@ public class DFSClient implements FSCons
   /**
    * Append to an existing file if {@link CreateFlag#APPEND} is present
    */
-  private OutputStream primitiveAppend(String src, EnumSet<CreateFlag> flag,
+  private DFSOutputStream primitiveAppend(String src, EnumSet<CreateFlag> flag,
       int buffersize, Progressable progress) throws IOException {
     if (flag.contains(CreateFlag.APPEND)) {
       HdfsFileStatus stat = getFileInfo(src);
@@ -674,7 +674,7 @@ public class DFSClient implements FSCons
       throws IOException, UnresolvedLinkException {
     checkOpen();
     CreateFlag.validate(flag);
-    OutputStream result = primitiveAppend(src, flag, buffersize, progress);
+    DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress);
     if (result == null) {
       result = new DFSOutputStream(this, src, absPermission,
           flag, createParent, replication, blockSize, progress, buffersize,
@@ -722,7 +722,7 @@ public class DFSClient implements FSCons
   }
 
   /** Method to get stream returned by append call */
-  private OutputStream callAppend(HdfsFileStatus stat, String src,
+  private DFSOutputStream callAppend(HdfsFileStatus stat, String src,
       int buffersize, Progressable progress) throws IOException {
     LocatedBlock lastBlock = null;
     try {
@@ -750,7 +750,7 @@ public class DFSClient implements FSCons
    * 
    * @see ClientProtocol#append(String, String) 
    */
-  OutputStream append(String src, int buffersize, Progressable progress) 
+  DFSOutputStream append(String src, int buffersize, Progressable progress) 
       throws IOException {
     checkOpen();
     HdfsFileStatus stat = getFileInfo(src);
@@ -758,7 +758,7 @@ public class DFSClient implements FSCons
       throw new FileNotFoundException("failed to append to non-existent file "
           + src + " on client " + clientName);
     }
-    OutputStream result = callAppend(stat, src, buffersize, progress);
+    final DFSOutputStream result = callAppend(stat, src, buffersize, progress);
     leaserenewer.put(src, result);
     return result;
   }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1099641&r1=1099640&r2=1099641&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Wed May 
4 23:52:03 2011
@@ -233,7 +233,7 @@ public class DistributedFileSystem exten
       Progressable progress) throws IOException {
 
     statistics.incrementWriteOps(1);
-    DFSOutputStream op = (DFSOutputStream)dfs.append(getPathName(f), bufferSize, progress);
+    final DFSOutputStream op = dfs.append(getPathName(f), bufferSize, progress);
     return new FSDataOutputStream(op, statistics, op.getInitialLen());
   }
 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java?rev=1099641&r1=1099640&r2=1099641&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/LeaseRenewer.java Wed May  4 23:52:03
2011
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.SocketTimeoutException;
-import java.util.SortedMap;
+import java.util.Map;
 import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
@@ -39,8 +39,8 @@ public class LeaseRenewer {
   /** A map from src -> DFSOutputStream of files that are currently being
    * written by this client.
    */
-  private final SortedMap<String, OutputStream> pendingCreates
-      = new TreeMap<String, OutputStream>();
+  private final Map<String, DFSOutputStream> filesBeingWritten
+      = new TreeMap<String, DFSOutputStream>();
   /** The time in milliseconds that the map became empty. */
   private long emptyTime = Long.MAX_VALUE;
   /** A fixed lease renewal time period in milliseconds */
@@ -98,7 +98,7 @@ public class LeaseRenewer {
         && System.currentTimeMillis() - emptyTime > gracePeriod;
   }
 
-  synchronized void put(String src, OutputStream out) {
+  synchronized void put(String src, DFSOutputStream out) {
     if (dfsclient.clientRunning) {
       if (daemon == null || isRenewerExpired()) {
         //start a new deamon with a new id.
@@ -118,14 +118,14 @@ public class LeaseRenewer {
         });
         daemon.start();
       }
-      pendingCreates.put(src, out);
+      filesBeingWritten.put(src, out);
       emptyTime = Long.MAX_VALUE;
     }
   }
   
   synchronized void remove(String src) {
-    pendingCreates.remove(src);
-    if (pendingCreates.isEmpty() && emptyTime == Long.MAX_VALUE) {
+    filesBeingWritten.remove(src);
+    if (filesBeingWritten.isEmpty() && emptyTime == Long.MAX_VALUE) {
       //discover the first time that the map is empty.
       emptyTime = System.currentTimeMillis();
     }
@@ -153,11 +153,11 @@ public class LeaseRenewer {
       String src;
       OutputStream out;
       synchronized (this) {
-        if (pendingCreates.isEmpty()) {
+        if (filesBeingWritten.isEmpty()) {
           return;
         }
-        src = pendingCreates.firstKey();
-        out = pendingCreates.remove(src);
+        src = filesBeingWritten.keySet().iterator().next();
+        out = filesBeingWritten.remove(src);
       }
       if (out != null) {
         try {
@@ -174,23 +174,23 @@ public class LeaseRenewer {
    */
   synchronized void abort() {
     dfsclient.clientRunning = false;
-    while (!pendingCreates.isEmpty()) {
-      String src = pendingCreates.firstKey();
-      DFSOutputStream out = (DFSOutputStream)pendingCreates.remove(src);
+    for(Map.Entry<String, DFSOutputStream> e : filesBeingWritten.entrySet()) {
+      final DFSOutputStream out = e.getValue();
       if (out != null) {
         try {
           out.abort();
         } catch (IOException ie) {
-          LOG.error("Exception aborting file " + src+ ": ", ie);
+          LOG.error("Failed to abort file " + e.getKey(), ie);
         }
       }
     }
+    filesBeingWritten.clear();
     RPC.stopProxy(dfsclient.rpcNamenode); // close connections to the namenode
   }
 
   private void renew() throws IOException {
     synchronized(this) {
-      if (pendingCreates.isEmpty()) {
+      if (filesBeingWritten.isEmpty()) {
         return;
       }
     }



Mime
View raw message