flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From se...@apache.org
Subject [2/2] flink git commit: [FLINK-2121] Fix the recursive summation in FileInputFormat.addFilesInDir
Date Mon, 01 Jun 2015 15:58:12 GMT
[FLINK-2121] Fix the recursive summation in FileInputFormat.addFilesInDir

This closes #752


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/03340919
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/03340919
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/03340919

Branch: refs/heads/master
Commit: 033409190235f93ed6d4e652214e7f35a34c3fe3
Parents: 7d29473
Author: Gabor Gevay <ggab90@gmail.com>
Authored: Sun May 31 13:27:15 2015 +0200
Committer: Stephan Ewen <sewen@apache.org>
Committed: Mon Jun 1 17:00:03 2015 +0200

----------------------------------------------------------------------
 .../org/apache/flink/api/common/io/FileInputFormat.java   | 10 ++++++----
 .../flink/api/common/io/EnumerateNestedFilesTest.java     | 10 +++++++++-
 2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/03340919/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
b/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
index 37739f5..a8f334b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
@@ -330,7 +330,7 @@ public abstract class FileInputFormat<OT> implements InputFormat<OT,
FileInputSp
 
 		// enumerate all files
 		if (file.isDir()) {
-			totalLength += addFilesInDir(file.getPath(), files, totalLength, false);
+			totalLength += addFilesInDir(file.getPath(), files, false);
 		} else {
 			files.add(file);
 			testForUnsplittable(file);
@@ -390,7 +390,7 @@ public abstract class FileInputFormat<OT> implements InputFormat<OT,
FileInputSp
 		final FileStatus pathFile = fs.getFileStatus(path);
 
 		if (pathFile.isDir()) {
-			totalLength += addFilesInDir(path, files, totalLength, true);
+			totalLength += addFilesInDir(path, files, true);
 		} else {
 			testForUnsplittable(pathFile);
 
@@ -497,14 +497,16 @@ public abstract class FileInputFormat<OT> implements InputFormat<OT,
FileInputSp
 	 * Enumerate all files in the directory and recursive if enumerateNestedFiles is true.
 	 * @return the total length of accepted files.
 	 */
-	private long addFilesInDir(Path path, List<FileStatus> files, long length, boolean
logExcludedFiles)
+	private long addFilesInDir(Path path, List<FileStatus> files, boolean logExcludedFiles)
 			throws IOException {
 		final FileSystem fs = path.getFileSystem();
 
+		long length = 0;
+
 		for(FileStatus dir: fs.listStatus(path)) {
 			if (dir.isDir()) {
 				if (acceptFile(dir) && enumerateNestedFiles) {
-					length += addFilesInDir(dir.getPath(), files, length, logExcludedFiles);
+					length += addFilesInDir(dir.getPath(), files, logExcludedFiles);
 				} else {
 					if (logExcludedFiles && LOG.isDebugEnabled()) {
 						LOG.debug("Directory "+dir.getPath().toString()+" did not pass the file-filter and
is excluded.");

http://git-wip-us.apache.org/repos/asf/flink/blob/03340919/flink-core/src/test/java/org/apache/flink/api/common/io/EnumerateNestedFilesTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/EnumerateNestedFilesTest.java
b/flink-core/src/test/java/org/apache/flink/api/common/io/EnumerateNestedFilesTest.java
index da4b0f1..d0caa22 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/EnumerateNestedFilesTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/EnumerateNestedFilesTest.java
@@ -330,10 +330,12 @@ public class EnumerateNestedFilesTest {
 			final long SIZE1 = 2077;
 			final long SIZE2 = 31909;
 			final long SIZE3 = 10;
-			final long TOTAL = SIZE1 + SIZE2 + SIZE3;
+			final long SIZE4 = 71;
+			final long TOTAL = SIZE1 + SIZE2 + SIZE3 + SIZE4;
 
 			String firstLevelDir = TestFileUtils.randomFileName();
 			String secondLevelDir = TestFileUtils.randomFileName();
+			String secondLevelDir2 = TestFileUtils.randomFileName();
 
 			File nestedDir = new File(tempPath + System.getProperty("file.separator") 
 					+ firstLevelDir);
@@ -345,10 +347,16 @@ public class EnumerateNestedFilesTest {
 			insideNestedDir.mkdirs();
 			insideNestedDir.deleteOnExit();
 
+			File insideNestedDir2 = new File(tempPath + System.getProperty("file.separator")
+					+ firstLevelDir + System.getProperty("file.separator") + secondLevelDir2);
+			insideNestedDir2.mkdirs();
+			insideNestedDir2.deleteOnExit();
+
 			// create a file in the first-level and two files in the nested dir
 			TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), SIZE1);
 			TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE2);
 			TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE3);
+			TestFileUtils.createTempFileInDirectory(insideNestedDir2.getAbsolutePath(), SIZE4);
 
 			this.format.setFilePath(new Path(nestedDir.toURI().toString()));
 			this.config.setBoolean("recursive.file.enumeration", true);


Mime
View raw message