Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A4CAF200CD9 for ; Thu, 20 Jul 2017 00:35:29 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A32E91689E5; Wed, 19 Jul 2017 22:35:29 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id CB2D01689C0 for ; Thu, 20 Jul 2017 00:35:28 +0200 (CEST) Received: (qmail 49854 invoked by uid 500); 19 Jul 2017 22:35:07 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 47076 invoked by uid 99); 19 Jul 2017 22:34:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Jul 2017 22:34:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2F203F5532; Wed, 19 Jul 2017 22:34:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xyao@apache.org To: common-commits@hadoop.apache.org Date: Wed, 19 Jul 2017 22:35:38 -0000 Message-Id: <09ee018008c445d486c0c7c69a812f33@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [46/50] [abbrv] hadoop git commit: HDFS-12139. HTTPFS liststatus returns incorrect pathSuffix for path of file. Contributed by Yongjun Zhang. archived-at: Wed, 19 Jul 2017 22:35:29 -0000 HDFS-12139. HTTPFS liststatus returns incorrect pathSuffix for path of file. Contributed by Yongjun Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3556e36b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3556e36b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3556e36b Branch: refs/heads/HDFS-7240 Commit: 3556e36be30211f46ac38899ce11a4d4cac6d635 Parents: 413b23e Author: Yongjun Zhang Authored: Wed Jul 19 10:54:13 2017 -0700 Committer: Yongjun Zhang Committed: Wed Jul 19 10:56:50 2017 -0700 ---------------------------------------------------------------------- .../hadoop/fs/http/server/FSOperations.java | 15 ++++++----- .../fs/http/client/BaseTestHttpFSWith.java | 26 +++++++++++++++++++- 2 files changed, 34 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3556e36b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 0fb665a..f1615c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -75,15 +75,17 @@ public class FSOperations { /** * @param fileStatuses list of FileStatus objects + * @param isFile is the fileStatuses from a file path * @return JSON map suitable for wire transport */ @SuppressWarnings({"unchecked"}) - private static Map toJson(FileStatus[] fileStatuses) { + private static Map toJson(FileStatus[] fileStatuses, + boolean isFile) { Map json = new LinkedHashMap<>(); Map inner = new LinkedHashMap<>(); JSONArray statuses = new JSONArray(); for (FileStatus f : fileStatuses) { - statuses.add(toJsonInner(f, false)); + statuses.add(toJsonInner(f, isFile)); } inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses); json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner); @@ -129,13 +131,14 @@ public class FSOperations { * These two classes are slightly different, due to the impedance * mismatches between the WebHDFS and FileSystem APIs. * @param entries + * @param isFile is the entries from a file path * @return json */ private static Map toJson(FileSystem.DirectoryEntries - entries) { + entries, boolean isFile) { Map json = new LinkedHashMap<>(); Map inner = new LinkedHashMap<>(); - Map fileStatuses = toJson(entries.getEntries()); + Map fileStatuses = toJson(entries.getEntries(), isFile); inner.put(HttpFSFileSystem.PARTIAL_LISTING_JSON, fileStatuses); inner.put(HttpFSFileSystem.REMAINING_ENTRIES_JSON, entries.hasMore() ? 1 : 0); @@ -690,7 +693,7 @@ public class FSOperations { @Override public Map execute(FileSystem fs) throws IOException { FileStatus[] fileStatuses = fs.listStatus(path, filter); - return toJson(fileStatuses); + return toJson(fileStatuses, fs.getFileStatus(path).isFile()); } @Override @@ -735,7 +738,7 @@ public class FSOperations { WrappedFileSystem wrappedFS = new WrappedFileSystem(fs); FileSystem.DirectoryEntries entries = wrappedFS.listStatusBatch(path, token); - return toJson(entries); + return toJson(entries, wrappedFS.getFileStatus(path).isFile()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3556e36b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 0fd3f91..e23093e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -364,8 +364,15 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { assertEquals(status2.getLen(), status1.getLen()); FileStatus[] stati = fs.listStatus(path.getParent()); - assertEquals(stati.length, 1); + assertEquals(1, stati.length); assertEquals(stati[0].getPath().getName(), path.getName()); + + // The full path should be the path to the file. See HDFS-12139 + FileStatus[] statl = fs.listStatus(path); + Assert.assertEquals(1, statl.length); + Assert.assertEquals(status2.getPath(), statl[0].getPath()); + Assert.assertEquals(statl[0].getPath().getName(), path.getName()); + Assert.assertEquals(stati[0].getPath(), statl[0].getPath()); } private static void assertSameListing(FileSystem expected, FileSystem @@ -411,6 +418,23 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { proxyFs.create(new Path(dir, "file" + i)).close(); assertSameListing(proxyFs, httpFs, dir); } + + // Test for HDFS-12139 + Path dir1 = new Path(getProxiedFSTestDir(), "dir1"); + proxyFs.mkdirs(dir1); + Path file1 = new Path(dir1, "file1"); + proxyFs.create(file1).close(); + + RemoteIterator si = proxyFs.listStatusIterator(dir1); + FileStatus statusl = si.next(); + FileStatus status = proxyFs.getFileStatus(file1); + Assert.assertEquals(file1.getName(), statusl.getPath().getName()); + Assert.assertEquals(status.getPath(), statusl.getPath()); + + si = proxyFs.listStatusIterator(file1); + statusl = si.next(); + Assert.assertEquals(file1.getName(), statusl.getPath().getName()); + Assert.assertEquals(status.getPath(), statusl.getPath()); } private void testWorkingdirectory() throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org