Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 74172200CAF for ; Thu, 22 Jun 2017 14:08:18 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 7101D160BE7; Thu, 22 Jun 2017 12:08:18 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8A90C160BE5 for ; Thu, 22 Jun 2017 14:08:17 +0200 (CEST) Received: (qmail 27668 invoked by uid 500); 22 Jun 2017 12:08:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 27659 invoked by uid 99); 22 Jun 2017 12:08:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Jun 2017 12:08:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 88678DFA84; Thu, 22 Jun 2017 12:08:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Message-Id: <5de7883b2c22493a8440b0a8cfae2174@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-11067. DFS#listStatusIterator(..) should throw FileNotFoundException if the directory deleted before fetching next batch of entries. Contributed by Vinayakumar B. Date: Thu, 22 Jun 2017 12:08:16 +0000 (UTC) archived-at: Thu, 22 Jun 2017 12:08:18 -0000 Repository: hadoop Updated Branches: refs/heads/trunk b64951905 -> 8dbd53ef9 HDFS-11067. DFS#listStatusIterator(..) should throw FileNotFoundException if the directory deleted before fetching next batch of entries. Contributed by Vinayakumar B. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8dbd53ef Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8dbd53ef Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8dbd53ef Branch: refs/heads/trunk Commit: 8dbd53ef9f34e3e05b159e4f5378e9c2c52c59c5 Parents: b649519 Author: Vinayakumar B Authored: Thu Jun 22 17:35:40 2017 +0530 Committer: Vinayakumar B Committed: Thu Jun 22 17:37:08 2017 +0530 ---------------------------------------------------------------------- .../src/site/markdown/filesystem/filesystem.md | 4 +++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 2 +- .../hadoop/hdfs/DistributedFileSystem.java | 2 +- .../org/apache/hadoop/hdfs/TestFileStatus.java | 27 ++++++++++++++++++-- 4 files changed, 31 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dbd53ef/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index b464941..b56666c 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -1185,6 +1185,10 @@ on (possibly remote) filesystems. These filesystems are invariably accessed concurrently; the state of the filesystem MAY change between a `hasNext()` probe and the invocation of the `next()` call. +During iteration through a `RemoteIterator`, if the directory is deleted on +remote filesystem, then `hasNext()` or `next()` call may throw +`FileNotFoundException`. + Accordingly, a robust iteration through a `RemoteIterator` would catch and discard `NoSuchElementException` exceptions raised during the process, which could be done through the `while(true)` iteration example above, or http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dbd53ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index 645f1ad..cd870ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -232,7 +232,7 @@ public class Hdfs extends AbstractFileSystem { thisListing = dfs.listPaths(src, thisListing.getLastName(), needLocation); if (thisListing == null) { - return false; // the directory is deleted + throw new FileNotFoundException("File " + src + " does not exist."); } i = 0; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dbd53ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 3e09804..f8af4ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1168,7 +1168,7 @@ public class DistributedFileSystem extends FileSystem { needLocation); statistics.incrementReadOps(1); if (thisListing == null) { - return false; + throw new FileNotFoundException("File " + p + " does not exist."); } i = 0; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dbd53ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java index c74bb63..31007dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java @@ -317,8 +317,31 @@ public class TestFileStatus { assertEquals(file3.toString(), itor.next().getPath().toString()); assertFalse(itor.hasNext()); - - fs.delete(dir, true); + itor = fs.listStatusIterator(dir); + assertEquals(dir3.toString(), itor.next().getPath().toString()); + assertEquals(dir4.toString(), itor.next().getPath().toString()); + fs.delete(dir.getParent(), true); + try { + itor.hasNext(); + fail("FileNotFoundException expected"); + } catch (FileNotFoundException fnfe) { + } + + fs.mkdirs(file2); + fs.mkdirs(dir3); + fs.mkdirs(dir4); + fs.mkdirs(dir5); + itor = fs.listStatusIterator(dir); + int count = 0; + try { + fs.delete(dir.getParent(), true); + while (itor.next() != null) { + count++; + } + fail("FileNotFoundException expected"); + } catch (FileNotFoundException fnfe) { + } + assertEquals(2, count); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org