Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8822C1802E for ; Wed, 1 Jul 2015 10:45:46 +0000 (UTC) Received: (qmail 12663 invoked by uid 500); 1 Jul 2015 10:45:34 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 11890 invoked by uid 500); 1 Jul 2015 10:45:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 10829 invoked by uid 99); 1 Jul 2015 10:45:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Jul 2015 10:45:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7FC5EE0508; Wed, 1 Jul 2015 10:45:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Wed, 01 Jul 2015 10:46:05 -0000 Message-Id: In-Reply-To: <83a8238545f945b08abdc9b5a131f831@git.apache.org> References: <83a8238545f945b08abdc9b5a131f831@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [34/50] hadoop git commit: HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina) HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d99017f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d99017f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d99017f Branch: refs/heads/YARN-2139 Commit: 6d99017f38f5a158b5cb65c74688b4c833e4e35f Parents: 77588e1 Author: Vinayakumar B Authored: Tue Jun 30 15:25:20 2015 +0530 Committer: Vinayakumar B Committed: Tue Jun 30 15:25:20 2015 +0530 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/fs/FileSystem.java | 6 ++--- .../org/apache/hadoop/hdfs/TestDFSShell.java | 26 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d99017f/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 50192ae..2cf9509 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -899,6 +899,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12089. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. (Duo Xu via cnauroth) + HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' + (J.Andreina via vinayakumarb) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d99017f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 1d7bc87..c98074a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2085,9 +2085,9 @@ public abstract class FileSystem extends Configured implements Closeable { /** Return the total size of all files in the filesystem.*/ public long getUsed() throws IOException{ long used = 0; - FileStatus[] files = listStatus(new Path("/")); - for(FileStatus file:files){ - used += file.getLen(); + RemoteIterator files = listFiles(new Path("/"), true); + while (files.hasNext()) { + used += files.next().getLen(); } return used; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d99017f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 2df31c4..1386124 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -913,6 +913,32 @@ public class TestDFSShell { cluster.shutdown(); } } + + @Test(timeout = 30000) + public void testTotalSizeOfAllFiles() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + FileSystem fs = cluster.getFileSystem(); + // create file under root + FSDataOutputStream File1 = fs.create(new Path("/File1")); + File1.write("hi".getBytes()); + File1.close(); + // create file under sub-folder + FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2")); + File2.write("hi".getBytes()); + File2.close(); + // getUsed() should return total length of all the files in Filesystem + assertEquals(4, fs.getUsed()); + } finally { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + } + private static void runCount(String path, long dirs, long files, FsShell shell ) throws IOException { ByteArrayOutputStream bytes = new ByteArrayOutputStream();