crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dbe...@apache.org
Subject git commit: CRUNCH-228: FileTargetImpl cuts off extensions of output files
Date Sun, 30 Jun 2013 16:16:53 GMT
Updated Branches:
  refs/heads/master 1082111c7 -> dea3fd93e


CRUNCH-228: FileTargetImpl cuts off extensions of output files


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/dea3fd93
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/dea3fd93
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/dea3fd93

Branch: refs/heads/master
Commit: dea3fd93eb54c68b7083fdbdb3f798aafb4d9e9a
Parents: 1082111
Author: Dave Beech <dbeech@apache.org>
Authored: Sun Jun 30 17:16:43 2013 +0100
Committer: Dave Beech <dbeech@apache.org>
Committed: Sun Jun 30 17:16:43 2013 +0100

----------------------------------------------------------------------
 .../apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java    | 8 ++++----
 .../main/java/org/apache/crunch/io/impl/FileTargetImpl.java  | 7 ++++---
 2 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/dea3fd93/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java
b/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java
index cd7fe0b..c6ea4fa 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java
@@ -91,7 +91,7 @@ public class TrevniKeyPipelineIT implements Serializable {
 
     Person person = genericCollection.materialize().iterator().next();
 
-    File trvFile = new File(outputFile, "part-m-00000-part-0.trv");
+    File trvFile = new File(outputFile, "part-m-00000.trv-part-0.trv");
 
     AvroColumnReader.Params params = new AvroColumnReader.Params(trvFile);
     params.setSchema(Person.SCHEMA$);
@@ -125,8 +125,8 @@ public class TrevniKeyPipelineIT implements Serializable {
 
     Person person = genericCollection.materialize().iterator().next();
 
-    File trv1File = new File(output1File, "part-m-00000-part-0.trv");
-    File trv2File = new File(output2File, "part-m-00000-part-0.trv");
+    File trv1File = new File(output1File, "part-m-00000.trv-part-0.trv");
+    File trv2File = new File(output2File, "part-m-00000.trv-part-0.trv");
 
     AvroColumnReader.Params params = new AvroColumnReader.Params(trv1File);
     params.setSchema(Person.SCHEMA$);
@@ -178,7 +178,7 @@ public class TrevniKeyPipelineIT implements Serializable {
 
     assertThat(retrievedPerson, is(person));
 
-    File trvFile = new File(outputFile, "part-m-00000-part-0.trv");
+    File trvFile = new File(outputFile, "part-m-00000.trv-part-0.trv");
 
     AvroColumnReader.Params params = new AvroColumnReader.Params(trvFile);
     params.setSchema(Person.SCHEMA$);

http://git-wip-us.apache.org/repos/asf/crunch/blob/dea3fd93/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java
index 50a1fd3..4d58830 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java
@@ -132,13 +132,14 @@ public class FileTargetImpl implements PathTarget {
   protected Path getDestFile(Configuration conf, Path src, Path dir, boolean mapOnlyJob)
       throws IOException {
     String outputFilename = null;
+    String sourceFilename = src.getName();
     if (mapOnlyJob) {
       outputFilename = getFileNamingScheme().getMapOutputName(conf, dir);
     } else {
-      outputFilename = getFileNamingScheme().getReduceOutputName(conf, dir, extractPartitionNumber(src.getName()));
+      outputFilename = getFileNamingScheme().getReduceOutputName(conf, dir, extractPartitionNumber(sourceFilename));
     }
-    if (src.getName().endsWith(org.apache.avro.mapred.AvroOutputFormat.EXT)) {
-      outputFilename += org.apache.avro.mapred.AvroOutputFormat.EXT;
+    if (sourceFilename.contains(".")) {
+      outputFilename += sourceFilename.substring(sourceFilename.indexOf("."));
     }
     return new Path(dir, outputFilename);
   }


Mime
View raw message