crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-590: Fix CSVInputFormat to work with S3
Date Tue, 02 Feb 2016 19:32:58 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 55d73bf17 -> 6463da421


CRUNCH-590: Fix CSVInputFormat to work with S3


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6463da42
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6463da42
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6463da42

Branch: refs/heads/master
Commit: 6463da4218568d3d3fd82a33930f8b7cc57c1288
Parents: 55d73bf
Author: Josh Wills <jwills@apache.org>
Authored: Tue Feb 2 11:05:04 2016 -0800
Committer: Josh Wills <jwills@apache.org>
Committed: Tue Feb 2 11:05:04 2016 -0800

----------------------------------------------------------------------
 .../main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/6463da42/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java b/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java
index 2894686..2dc5e13 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java
@@ -92,10 +92,10 @@ public class CSVInputFormat extends FileInputFormat<LongWritable, Text>
implemen
     final long splitSize = job.getConfiguration().getLong(CSVFileSource.INPUT_SPLIT_SIZE,
67108864);
     final List<InputSplit> splits = new ArrayList<InputSplit>();
     final Path[] paths = FileUtil.stat2Paths(listStatus(job).toArray(new FileStatus[0]));
-    final FileSystem fileSystem = FileSystem.get(job.getConfiguration());
     FSDataInputStream inputStream = null;
     try {
       for (final Path path : paths) {
+        FileSystem fileSystem = path.getFileSystem(job.getConfiguration());
         inputStream = fileSystem.open(path);
         splits.addAll(getSplitsForFile(splitSize, fileSystem.getFileStatus(path).getLen(),
path, inputStream));
       }
@@ -199,4 +199,4 @@ public class CSVInputFormat extends FileInputFormat<LongWritable, Text>
implemen
 
     return splitsList;
   }
-}
\ No newline at end of file
+}


Mime
View raw message