hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r685405 - in /hadoop/core/trunk: ./ lib/ src/core/org/apache/hadoop/fs/kfs/ src/test/org/apache/hadoop/fs/kfs/
Date Wed, 13 Aug 2008 01:08:10 GMT
Author: omalley
Date: Tue Aug 12 18:08:09 2008
New Revision: 685405

URL: http://svn.apache.org/viewvc?rev=685405&view=rev
Log:
HADOOP-3816. Faster directory listing in KFS. (Sriram Rao via omalley)

Added:
    hadoop/core/trunk/lib/kfs-0.2.LICENSE.txt
      - copied unchanged from r685289, hadoop/core/trunk/lib/kfs-0.1.LICENSE.txt
Removed:
    hadoop/core/trunk/lib/kfs-0.1.3.jar
    hadoop/core/trunk/lib/kfs-0.1.LICENSE.txt
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/IFSImpl.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KFSImpl.java
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/kfs/KFSEmulationImpl.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=685405&r1=685404&r2=685405&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Aug 12 18:08:09 2008
@@ -204,6 +204,8 @@
     HADOOP-3864. Prevent the JobTracker from locking up when a job is being
     initialized. (acmurthy via omalley)
 
+    HADOOP-3816. Faster directory listing in KFS. (Sriram Rao via omalley)
+
   BUG FIXES
 
     HADOOP-3563.  Refactor the distributed upgrade code so that it is 

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/IFSImpl.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/IFSImpl.java?rev=685405&r1=685404&r2=685405&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/IFSImpl.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/IFSImpl.java Tue Aug 12 18:08:09 2008
@@ -31,12 +31,15 @@
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 
 interface IFSImpl {
     public boolean exists(String path) throws IOException;
     public boolean isDirectory(String path) throws IOException;
     public boolean isFile(String path) throws IOException;
     public String[] readdir(String path) throws IOException;
+    public FileStatus[] readdirplus(Path path) throws IOException;
 
     public int mkdirs(String path) throws IOException;
     public int rename(String source, String dest) throws IOException;

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KFSImpl.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KFSImpl.java?rev=685405&r1=685404&r2=685405&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KFSImpl.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KFSImpl.java Tue Aug 12 18:08:09 2008
@@ -24,8 +24,11 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 
 import org.kosmix.kosmosfs.access.KfsAccess;
+import org.kosmix.kosmosfs.access.KfsFileAttr;
 
 class KFSImpl implements IFSImpl {
     private KfsAccess kfsAccess = null;
@@ -59,6 +62,40 @@
         return kfsAccess.kfs_readdir(path);
     }
 
+    public FileStatus[] readdirplus(Path path) throws IOException {
+        String srep = path.toUri().getPath();
+        KfsFileAttr[] fattr = kfsAccess.kfs_readdirplus(srep);
+        if (fattr == null)
+            return null;
+        int numEntries = 0;
+        for (int i = 0; i < fattr.length; i++) {
+            if ((fattr[i].filename.compareTo(".") == 0) || (fattr[i].filename.compareTo("..")
== 0))
+                continue;
+            numEntries++;
+        }
+        FileStatus[] fstatus = new FileStatus[numEntries];
+        int j = 0;
+        for (int i = 0; i < fattr.length; i++) {
+            if ((fattr[i].filename.compareTo(".") == 0) || (fattr[i].filename.compareTo("..")
== 0))
+                continue;
+            Path fn = new Path(path, fattr[i].filename);
+
+            if (fattr[i].isDirectory)
+                fstatus[j] = new FileStatus(0, true, 1, 0, fattr[i].modificationTime, fn);
+            else
+                fstatus[j] = new FileStatus(fattr[i].filesize, fattr[i].isDirectory,
+                                            fattr[i].replication,
+                                            (long)
+                                            (1 << 26),
+                                            fattr[i].modificationTime,
+                                            fn);
+
+            j++;
+        }
+        return fstatus;
+    }
+
+
     public int mkdirs(String path) throws IOException {
         return kfsAccess.kfs_mkdirs(path);
     }

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=685405&r1=685404&r2=685405&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Tue Aug 12 18:08:09
2008
@@ -128,36 +128,13 @@
     }
 
     public FileStatus[] listStatus(Path path) throws IOException {
-	Path absolute = makeAbsolute(path);
+        Path absolute = makeAbsolute(path);
         String srep = absolute.toUri().getPath();
 
-	if (kfsImpl.isFile(srep))
-	    return new FileStatus[] { getFileStatus(path) } ;
-
-	String[] entries = kfsImpl.readdir(srep);
-
-        if (entries == null)
-            return null;
+        if (kfsImpl.isFile(srep))
+                return new FileStatus[] { getFileStatus(path) } ;
 
-        // kfsreaddir() returns "." and ".."; strip them before
-        // passing back to hadoop fs.
-	int numEntries = 0;
-	for (int i = 0; i < entries.length; i++) {
-	    if ((entries[i].compareTo(".") == 0) || (entries[i].compareTo("..") == 0))
-		continue;
-	    numEntries++;
-	}
-
-	FileStatus[] pathEntries = new FileStatus[numEntries];
-	int j = 0;
-	for (int i = 0; i < entries.length; i++) {
-	    if ((entries[i].compareTo(".") == 0) || (entries[i].compareTo("..") == 0))
-		continue;
-
-	    pathEntries[j] = getFileStatus(new Path(path, entries[i]));
-	    j++;
-	}
-	return pathEntries;
+        return kfsImpl.readdirplus(absolute);
     }
 
     public FileStatus getFileStatus(Path path) throws IOException {

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/kfs/KFSEmulationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/kfs/KFSEmulationImpl.java?rev=685405&r1=685404&r2=685405&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/kfs/KFSEmulationImpl.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/kfs/KFSEmulationImpl.java Tue Aug 12 18:08:09
2008
@@ -65,6 +65,10 @@
         return entries;
     }
 
+    public FileStatus[] readdirplus(Path path) throws IOException {
+        return localFS.listStatus(path);
+    }
+
     public int mkdirs(String path) throws IOException {
         if (localFS.mkdirs(new Path(path)))
             return 0;



Mime
View raw message