Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 24978 invoked from network); 2 Aug 2010 20:27:06 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 2 Aug 2010 20:27:06 -0000 Received: (qmail 75887 invoked by uid 500); 2 Aug 2010 20:27:06 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 75826 invoked by uid 500); 2 Aug 2010 20:27:05 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 75819 invoked by uid 99); 2 Aug 2010 20:27:05 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Aug 2010 20:27:05 +0000 X-ASF-Spam-Status: No, hits=-1998.0 required=10.0 tests=ALL_TRUSTED,FB_GET_MEDS X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Aug 2010 20:27:02 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 9E9242388A38; Mon, 2 Aug 2010 20:25:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r981676 - in /hadoop/common/trunk/src: java/org/apache/hadoop/fs/ test/core/org/apache/hadoop/fs/ Date: Mon, 02 Aug 2010 20:25:44 -0000 To: common-commits@hadoop.apache.org From: hairong@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100802202544.9E9242388A38@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: hairong Date: Mon Aug 2 20:25:44 2010 New Revision: 981676 URL: http://svn.apache.org/viewvc?rev=981676&view=rev Log: HADOOP-6890. Part 2: Incoport the change made in FileContext into FileSystem. Contributed by Hairong Kuang. Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestListFiles.java Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=981676&r1=981675&r2=981676&view=diff ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java Mon Aug 2 20:25:44 2010 @@ -20,6 +20,7 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.Arrays; +import java.util.Iterator; import java.util.zip.CRC32; import org.apache.commons.logging.Log; @@ -493,6 +494,21 @@ public abstract class ChecksumFileSystem return fs.listStatus(f, DEFAULT_FILTER); } + /** + * List the statuses of the files/directories in the given path if the path is + * a directory. + * + * @param f + * given path + * @return the statuses of the files/directories in the given patch + * @throws IOException + */ + @Override + public Iterator listLocatedStatus(Path f) + throws IOException { + return fs.listLocatedStatus(f, DEFAULT_FILTER); + } + @Override public boolean mkdirs(Path f) throws IOException { return fs.mkdirs(f); Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=981676&r1=981675&r2=981676&view=diff ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Mon Aug 2 20:25:44 2010 @@ -1644,16 +1644,16 @@ public final class FileContext { */ @Override public boolean hasNext() { - while (curFile == null) { - if (curItor.hasNext()) { - handleFileStat(curItor.next()); - } else if (!itors.empty()) { - curItor = itors.pop(); - } else { - return false; - } + while (curFile == null) { + if (curItor.hasNext()) { + handleFileStat(curItor.next()); + } else if (!itors.empty()) { + curItor = itors.pop(); + } else { + return false; } - return true; + } + return true; } /** Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=981676&r1=981675&r2=981676&view=diff ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Mon Aug 2 20:25:44 2010 @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; @@ -1320,13 +1319,95 @@ public abstract class FileSystem extends } /** + * List the statuses of the files/directories in the given path if the path is + * a directory. + * Return the file's status and block locations If the path is a file. + * + * If a returned status is a file, it contains the file's block locations. + * + * @param f is the path + * @param filter path filter + * + * @return an iterator that traverses statuses of the files/directories + * in the given path + * If any IO exception (for example the input directory gets deleted while + * listing is being executed), next() or hasNext() of the returned iterator + * may throw a RuntimeException with the IO exception as the cause. + * + * @throws FileNotFoundException If f does not exist + * @throws IOException If an I/O error occurred + */ + public Iterator listLocatedStatus(final Path f) + throws FileNotFoundException, IOException { + return listLocatedStatus(f, DEFAULT_FILTER); + } + + /** + * Listing a directory + * The returned results include its block location if it is a file + * The results are filtered by the given path filter + * @param f a path + * @param filter a path filter + * @return an iterator that traverses statuses of the files/directories + * in the given path + * @throws FileNotFoundException if f does not exist + * @throws IOException if any I/O error occurred + */ + protected Iterator listLocatedStatus(final Path f, + final PathFilter filter) + throws FileNotFoundException, IOException { + return new Iterator() { + private final FileStatus[] stats = listStatus(f, filter); + private int i = 0; + + /** + * {@inheritDoc} + * @return {@inheritDog} + * @throws Runtimeexception if any IOException occurs during traversal; + * the IOException is set as the cause of the RuntimeException + */ + @Override + public boolean hasNext() { + return i() { - private LinkedList fileStats = new LinkedList(); - private Stack dirStats = new Stack(); - - { // initializer - list(f); - } - + private Stack> itors = + new Stack>(); + Iterator curItor = listLocatedStatus(f); + LocatedFileStatus curFile; + /** * {@inheritDoc} * @return {@inheritDog} @@ -1358,42 +1437,40 @@ public abstract class FileSystem extends */ @Override public boolean hasNext() { - if (fileStats.isEmpty()) { - listDir(); - } - return !fileStats.isEmpty(); - } - - /** - * list at least one directory until file list is not empty - */ - private void listDir() { - while (fileStats.isEmpty() && !dirStats.isEmpty()) { - FileStatus dir = dirStats.pop(); - list(dir.getPath()); + while (curFile == null) { + if (curItor.hasNext()) { + handleFileStat(curItor.next()); + } else if (!itors.empty()) { + curItor = itors.pop(); + } else { + return false; + } } + return true; } /** - * List the given path - * - * @param dirPath a path + * Process the input stat. + * If it is a file, return the file stat. + * If it is a directory, tranverse the directory if recursive is true; + * ignore it if recursive is false. + * @param stat input status + * @throws RuntimeException if any io error occurs; the io exception + * is set as the cause of RuntimeException */ - private void list(Path dirPath) { + private void handleFileStat(LocatedFileStatus stat) { try { - FileStatus[] stats = listStatus(dirPath); - for (FileStatus stat : stats) { - if (stat.isFile()) { - fileStats.add(stat); - } else if (recursive) { // directory & recursive - dirStats.push(stat); - } + if (stat.isFile()) { // file + curFile = stat; + } else if (recursive) { // directory + itors.push(curItor); + curItor = listLocatedStatus(stat.getPath()); } } catch (IOException ioe) { - throw (RuntimeException) new RuntimeException().initCause(ioe); - } + throw (RuntimeException)new RuntimeException().initCause(ioe); + } } - + /** * {@inheritDoc} * @return {@inheritDoc} @@ -1403,19 +1480,14 @@ public abstract class FileSystem extends */ @Override public LocatedFileStatus next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - FileStatus status = fileStats.remove(); - try { - BlockLocation[] locs = getFileBlockLocations( - status, 0, status.getLen()); - return new LocatedFileStatus(status, locs); - } catch (IOException ioe) { - throw (RuntimeException) new RuntimeException().initCause(ioe); - } + if (hasNext()) { + LocatedFileStatus result = curFile; + curFile = null; + return result; + } + throw new java.util.NoSuchElementException("No more entry in " + f); } - + @Override public void remove() { throw new UnsupportedOperationException("Remove is not supported"); Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=981676&r1=981675&r2=981676&view=diff ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Mon Aug 2 20:25:44 2010 @@ -21,6 +21,7 @@ package org.apache.hadoop.fs; import java.io.*; import java.net.URI; import java.util.EnumSet; +import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -165,6 +166,12 @@ public class FilterFileSystem extends Fi return fs.listStatus(f); } + /** List files and its block locations in a directory. */ + public Iterator listLocatedStatus(Path f) + throws IOException { + return fs.listLocatedStatus(f); + } + public Path getHomeDirectory() { return fs.getHomeDirectory(); } Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java?rev=981676&r1=981675&r2=981676&view=diff ============================================================================== --- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java (original) +++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestFilterFileSystem.java Mon Aug 2 20:25:44 2010 @@ -115,6 +115,13 @@ public class TestFilterFileSystem extend final Path path, final boolean isRecursive) { return null; } + public Iterator listLocatedStatus(Path f) { + return null; + } + public Iterator listLocatedStatus(Path f, + final PathFilter filter) { + return null; + } public void copyFromLocalFile(Path src, Path dst) { } public void moveFromLocalFile(Path[] srcs, Path dst) { } public void moveFromLocalFile(Path src, Path dst) { } Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestListFiles.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestListFiles.java?rev=981676&r1=981675&r2=981676&view=diff ============================================================================== --- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestListFiles.java (original) +++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestListFiles.java Mon Aug 2 20:25:44 2010 @@ -138,13 +138,13 @@ public class TestListFiles { itor = fs.listFiles(TEST_DIR, true); stat = itor.next(); assertTrue(stat.isFile()); - assertEquals(fs.makeQualified(FILE1), stat.getPath()); - stat = itor.next(); - assertTrue(stat.isFile()); assertEquals(fs.makeQualified(FILE2), stat.getPath()); stat = itor.next(); assertTrue(stat.isFile()); assertEquals(fs.makeQualified(FILE3), stat.getPath()); + stat = itor.next(); + assertTrue(stat.isFile()); + assertEquals(fs.makeQualified(FILE1), stat.getPath()); assertFalse(itor.hasNext()); itor = fs.listFiles(TEST_DIR, false);