incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [4/4] git commit: CRUNCH-24: Clean up test suite output.
Date Thu, 26 Jul 2012 05:48:26 GMT
CRUNCH-24: Clean up test suite output.

Add new junit @Rule TemporaryPath that provides a temporary directory.
The directory and all files inside are deleted after a test run.
Change integration tests to use TemporaryPath where necessary.

Signed-off-by: jwills <jwills@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/6f549ed3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/6f549ed3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/6f549ed3

Branch: refs/heads/master
Commit: 6f549ed3ed0267b0872b5d3256a6f313a1a412c8
Parents: b645a55
Author: Matthias Friedrich <matt@mafr.de>
Authored: Sun Jul 22 09:27:15 2012 +0200
Committer: jwills <jwills@apache.org>
Committed: Wed Jul 25 22:23:37 2012 -0700

----------------------------------------------------------------------
 .../java/org/apache/crunch/MultipleOutputIT.java   |   16 +--
 .../org/apache/crunch/PCollectionGetSizeIT.java    |   11 +-
 .../it/java/org/apache/crunch/TermFrequencyIT.java |   11 +-
 crunch/src/it/java/org/apache/crunch/TfIdfIT.java  |   33 ++---
 .../org/apache/crunch/TupleNClassCastBugIT.java    |   21 ++--
 .../java/org/apache/crunch/WordCountHBaseIT.java   |   14 +--
 .../src/it/java/org/apache/crunch/WordCountIT.java |   19 ++--
 .../crunch/impl/mr/collect/UnionCollectionIT.java  |   35 +++---
 .../it/java/org/apache/crunch/lib/CogroupIT.java   |   13 +-
 .../java/org/apache/crunch/test/TemporaryPath.java |  105 +++++++++++++++
 .../org/apache/crunch/test/TemporaryPathIT.java    |   59 ++++++++
 11 files changed, 244 insertions(+), 93 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java b/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
index 7f50b54..90925ba 100644
--- a/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
+++ b/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
@@ -27,15 +27,18 @@ import java.util.List;
 
 import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.At;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.avro.AvroTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.io.Files;
 
 public class MultipleOutputIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   public static PCollection<String> evenCountLetters(PCollection<String> words,
PTypeFamily typeFamily) {
     return words.parallelDo("even", new FilterFn<String>() {
@@ -79,11 +82,9 @@ public class MultipleOutputIT {
   }
 
   public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
-    String inputPath = FileHelper.createTempCopyOf("letters.txt");
-    File outputEven = FileHelper.createOutputPath();
-    File outputOdd = FileHelper.createOutputPath();
-    String outputPathEven = outputEven.getAbsolutePath();
-    String outputPathOdd = outputOdd.getAbsolutePath();
+    String inputPath = tmpDir.copyResourceFileName("letters.txt");
+    String outputPathEven = tmpDir.getFileName("even");
+    String outputPathOdd = tmpDir.getFileName("odd");
 
     PCollection<String> words = pipeline.read(At.textFile(inputPath, typeFamily.strings()));
 
@@ -96,9 +97,6 @@ public class MultipleOutputIT {
 
     checkFileContents(outputPathEven, Arrays.asList("bb"));
     checkFileContents(outputPathOdd, Arrays.asList("a"));
-
-    outputEven.deleteOnExit();
-    outputOdd.deleteOnExit();
   }
 
   private void checkFileContents(String filePath, List<String> expected) throws IOException
{

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java b/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java
index a6571bb..4ff8490 100644
--- a/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java
+++ b/crunch/src/it/java/org/apache/crunch/PCollectionGetSizeIT.java
@@ -28,12 +28,15 @@ import java.io.IOException;
 
 import org.apache.crunch.impl.mem.MemPipeline;
 import org.apache.crunch.impl.mr.MRPipeline;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.junit.Before;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
 
 public class PCollectionGetSizeIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   private String emptyInputPath;
   private String nonEmptyInputPath;
@@ -51,9 +54,9 @@ public class PCollectionGetSizeIT {
 
   @Before
   public void setUp() throws IOException {
-    emptyInputPath = FileHelper.createTempCopyOf("emptyTextFile.txt");
-    nonEmptyInputPath = FileHelper.createTempCopyOf("set1.txt");
-    outputPath = FileHelper.createOutputPath().getAbsolutePath();
+    emptyInputPath = tmpDir.copyResourceFileName("emptyTextFile.txt");
+    nonEmptyInputPath = tmpDir.copyResourceFileName("set1.txt");
+    outputPath = tmpDir.getFileName("output");
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java b/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java
index e0d0b47..ee2267d 100644
--- a/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java
+++ b/crunch/src/it/java/org/apache/crunch/TermFrequencyIT.java
@@ -29,13 +29,16 @@ import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.At;
 import org.apache.crunch.io.ReadableSourceTarget;
 import org.apache.crunch.lib.Aggregate;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
 import org.junit.Test;
 
 @SuppressWarnings("serial")
 public class TermFrequencyIT implements Serializable {
+  @Rule
+  public transient TemporaryPath tmpDir = new TemporaryPath();
 
   @Test
   public void testTermFrequencyWithNoTransform() throws IOException {
@@ -58,10 +61,10 @@ public class TermFrequencyIT implements Serializable {
   }
 
   public void run(Pipeline pipeline, PTypeFamily typeFamily, boolean transformTF) throws
IOException {
-    String input = FileHelper.createTempCopyOf("docs.txt");
+    String input = tmpDir.copyResourceFileName("docs.txt");
 
-    File transformedOutput = FileHelper.createOutputPath();
-    File tfOutput = FileHelper.createOutputPath();
+    File transformedOutput = tmpDir.getFile("transformed-output");
+    File tfOutput = tmpDir.getFile("tf-output");
 
     PCollection<String> docs = pipeline.readTextFile(input);
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/TfIdfIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/TfIdfIT.java b/crunch/src/it/java/org/apache/crunch/TfIdfIT.java
index f48de8c..de1d5dc 100644
--- a/crunch/src/it/java/org/apache/crunch/TfIdfIT.java
+++ b/crunch/src/it/java/org/apache/crunch/TfIdfIT.java
@@ -17,8 +17,6 @@
  */
 package org.apache.crunch;
 
-import static com.google.common.io.Resources.getResource;
-import static com.google.common.io.Resources.newInputStreamSupplier;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -33,9 +31,11 @@ import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.seq.SeqFileSourceTarget;
 import org.apache.crunch.lib.Aggregate;
 import org.apache.crunch.lib.Join;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
 import org.apache.hadoop.fs.Path;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
@@ -43,6 +43,9 @@ import com.google.common.io.Files;
 
 @SuppressWarnings("serial")
 public class TfIdfIT implements Serializable {
+  @Rule
+  public transient TemporaryPath tmpDir = new TemporaryPath();
+
   // total number of documents, should calculate
   protected static final double N = 2;
 
@@ -169,16 +172,13 @@ public class TfIdfIT implements Serializable {
   }
 
   public void run(Pipeline pipeline, PTypeFamily typeFamily, boolean singleRun) throws IOException
{
-    File input = File.createTempFile("docs", "txt");
-    input.deleteOnExit();
-    Files.copy(newInputStreamSupplier(getResource("docs.txt")), input);
-
-    String outputPath1 = getOutput();
-    String outputPath2 = getOutput();
+    String inputFile = tmpDir.copyResourceFileName("docs.txt");
+    String outputPath1 = tmpDir.getFileName("output1");
+    String outputPath2 = tmpDir.getFileName("output2");
 
-    Path tfPath = new Path(getOutput("termfreq"));
+    Path tfPath = tmpDir.getPath("termfreq");
 
-    PCollection<String> docs = pipeline.readTextFile(input.getAbsolutePath());
+    PCollection<String> docs = pipeline.readTextFile(inputFile);
 
     PTable<String, Collection<Pair<String, Double>>> results = generateTFIDF(docs,
tfPath, typeFamily);
     pipeline.writeTextFile(results, outputPath1);
@@ -198,7 +198,6 @@ public class TfIdfIT implements Serializable {
 
     // Check the lowercase version...
     File outputFile = new File(outputPath1, "part-r-00000");
-    outputFile.deleteOnExit();
     List<String> lines = Files.readLines(outputFile, Charset.defaultCharset());
     boolean passed = false;
     for (String line : lines) {
@@ -211,7 +210,6 @@ public class TfIdfIT implements Serializable {
 
     // ...and the uppercase version
     outputFile = new File(outputPath2, "part-r-00000");
-    outputFile.deleteOnExit();
     lines = Files.readLines(outputFile, Charset.defaultCharset());
     passed = false;
     for (String line : lines) {
@@ -222,15 +220,4 @@ public class TfIdfIT implements Serializable {
     }
     assertTrue(passed);
   }
-
-  public static String getOutput() throws IOException {
-    return getOutput("output");
-  }
-
-  public static String getOutput(String prefix) throws IOException {
-    File output = File.createTempFile(prefix, "");
-    String path = output.getAbsolutePath();
-    output.delete();
-    return path;
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java b/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java
index 39919ba..f845214 100644
--- a/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java
+++ b/crunch/src/it/java/org/apache/crunch/TupleNClassCastBugIT.java
@@ -17,8 +17,6 @@
  */
 package org.apache.crunch;
 
-import static com.google.common.io.Resources.getResource;
-import static com.google.common.io.Resources.newInputStreamSupplier;
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
@@ -27,14 +25,19 @@ import java.nio.charset.Charset;
 import java.util.List;
 
 import org.apache.crunch.impl.mr.MRPipeline;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.avro.AvroTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.io.Files;
 
+
 public class TupleNClassCastBugIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   public static PCollection<TupleN> mapGroupDo(PCollection<String> lines, PTypeFamily
ptf) {
     PTable<String, TupleN> mapped = lines.parallelDo(new MapFn<String, Pair<String,
TupleN>>() {
@@ -68,15 +71,10 @@ public class TupleNClassCastBugIT {
   }
 
   public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
-    File input = File.createTempFile("docs", "txt");
-    input.deleteOnExit();
-    Files.copy(newInputStreamSupplier(getResource("docs.txt")), input);
-
-    File output = File.createTempFile("output", "");
-    String outputPath = output.getAbsolutePath();
-    output.delete();
+    String inputPath = tmpDir.copyResourceFileName("docs.txt");
+    String outputPath = tmpDir.getFileName("output");
 
-    PCollection<String> docLines = pipeline.readTextFile(input.getAbsolutePath());
+    PCollection<String> docLines = pipeline.readTextFile(inputPath);
     pipeline.writeTextFile(mapGroupDo(docLines, typeFamily), outputPath);
     pipeline.done();
 
@@ -85,13 +83,12 @@ public class TupleNClassCastBugIT {
     // *** which is thrown in a different thread during the reduce phase. If all
     // is well
     // *** the file will exist and have six lines. Otherwise the bug is present.
-    File outputFile = new File(output, "part-r-00000");
+    File outputFile = new File(outputPath, "part-r-00000");
     List<String> lines = Files.readLines(outputFile, Charset.defaultCharset());
     int lineCount = 0;
     for (String line : lines) {
       lineCount++;
     }
     assertEquals(6, lineCount);
-    output.deleteOnExit();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
index 9ff171d..9704c3b 100644
--- a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
+++ b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
@@ -28,12 +28,11 @@ import java.util.Random;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.hbase.HBaseSourceTarget;
 import org.apache.crunch.io.hbase.HBaseTarget;
 import org.apache.crunch.lib.Aggregate;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.writable.Writables;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
@@ -52,12 +51,14 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapred.TaskAttemptContext;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.io.ByteStreams;
 
 public class WordCountHBaseIT {
-  protected static final Log LOG = LogFactory.getLog(WordCountHBaseIT.class);
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   private static final byte[] COUNTS_COLFAM = Bytes.toBytes("cf");
   private static final byte[] WORD_COLFAM = Bytes.toBytes("cf");
@@ -88,15 +89,10 @@ public class WordCountHBaseIT {
     }, Writables.writables(Put.class));
   }
 
-  @SuppressWarnings("deprecation")
   @Before
   public void setUp() throws Exception {
     Configuration conf = hbaseTestUtil.getConfiguration();
-    File tmpDir = File.createTempFile("logdir", "");
-    tmpDir.delete();
-    tmpDir.mkdir();
-    tmpDir.deleteOnExit();
-    conf.set("hadoop.log.dir", tmpDir.getAbsolutePath());
+    conf.set("hadoop.log.dir", tmpDir.getRootFileName());
     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
     conf.setInt("hbase.master.info.port", -1);
     conf.setInt("hbase.regionserver.info.port", -1);

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/WordCountIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/WordCountIT.java b/crunch/src/it/java/org/apache/crunch/WordCountIT.java
index ba457ee..5ba931f 100644
--- a/crunch/src/it/java/org/apache/crunch/WordCountIT.java
+++ b/crunch/src/it/java/org/apache/crunch/WordCountIT.java
@@ -29,10 +29,11 @@ import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.At;
 import org.apache.crunch.io.To;
 import org.apache.crunch.lib.Aggregate;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.avro.AvroTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.collect.ImmutableList;
@@ -40,6 +41,8 @@ import com.google.common.collect.Lists;
 import com.google.common.io.Files;
 
 public class WordCountIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   enum WordCountStats {
     ANDS
@@ -113,9 +116,9 @@ public class WordCountIT {
     runWithTop(AvroTypeFamily.getInstance());
   }
 
-  public static void runWithTop(PTypeFamily tf) throws IOException {
+  public void runWithTop(PTypeFamily tf) throws IOException {
     Pipeline pipeline = new MRPipeline(WordCountIT.class);
-    String inputPath = FileHelper.createTempCopyOf("shakes.txt");
+    String inputPath = tmpDir.copyResourceFileName("shakes.txt");
 
     PCollection<String> shakespeare = pipeline.read(At.textFile(inputPath, tf.strings()));
     PTable<String, Long> wordCount = wordCount(shakespeare, tf);
@@ -126,9 +129,8 @@ public class WordCountIT {
   }
 
   public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
-    String inputPath = FileHelper.createTempCopyOf("shakes.txt");
-    File output = FileHelper.createOutputPath();
-    String outputPath = output.getAbsolutePath();
+    String inputPath = tmpDir.copyResourceFileName("shakes.txt");
+    String outputPath = tmpDir.getFileName("output");
 
     PCollection<String> shakespeare = pipeline.read(At.textFile(inputPath, typeFamily.strings()));
     PTable<String, Long> wordCount = wordCount(shakespeare, typeFamily);
@@ -139,9 +141,7 @@ public class WordCountIT {
     }
 
     if (runSecond) {
-      File substrCount = File.createTempFile("substr", "");
-      String substrPath = substrCount.getAbsolutePath();
-      substrCount.delete();
+      String substrPath = tmpDir.getFileName("substr");
       PTable<String, Long> we = substr(wordCount).groupByKey().combineValues(CombineFn.<String>
SUM_LONGS());
       pipeline.writeTextFile(we, substrPath);
     }
@@ -165,6 +165,5 @@ public class WordCountIT {
       }
     }
     assertTrue(passed);
-    output.deleteOnExit();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java b/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java
index 1bb3305..9748143 100644
--- a/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java
+++ b/crunch/src/it/java/org/apache/crunch/impl/mr/collect/UnionCollectionIT.java
@@ -36,13 +36,14 @@ import org.apache.crunch.impl.mem.MemPipeline;
 import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.At;
 import org.apache.crunch.io.To;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.avro.AvroTypeFamily;
 import org.apache.crunch.types.avro.Avros;
 import org.apache.crunch.types.writable.WritableTypeFamily;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -52,6 +53,8 @@ import com.google.common.collect.Lists;
 
 @RunWith(value = Parameterized.class)
 public class UnionCollectionIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   private static final Log LOG = LogFactory.getLog(UnionCollectionIT.class);
 
@@ -64,8 +67,8 @@ public class UnionCollectionIT {
   @Before
   @SuppressWarnings("unchecked")
   public void setUp() throws IOException {
-    String inputFile1 = FileHelper.createTempCopyOf("set1.txt");
-    String inputFile2 = FileHelper.createTempCopyOf("set2.txt");
+    String inputFile1 = tmpDir.copyResourceFileName("set1.txt");
+    String inputFile2 = tmpDir.copyResourceFileName("set2.txt");
 
     PCollection<String> firstCollection = pipeline.read(At.textFile(inputFile1, typeFamily.strings()));
     PCollection<String> secondCollection = pipeline.read(At.textFile(inputFile2, typeFamily.strings()));
@@ -114,30 +117,30 @@ public class UnionCollectionIT {
   @Test
   public void unionWriteShouldNotThrowNPE() throws IOException {
 
-    File outputPath1 = FileHelper.createOutputPath();
-    File outputPath2 = FileHelper.createOutputPath();
-    File outputPath3 = FileHelper.createOutputPath();
+    String outputPath1 = tmpDir.getFileName("output1");
+    String outputPath2 = tmpDir.getFileName("output2");
+    String outputPath3 = tmpDir.getFileName("output3");
 
     if (typeFamily == AvroTypeFamily.getInstance()) {
-      union.write(To.avroFile(outputPath1.getAbsolutePath()));
-      pipeline.write(union, To.avroFile(outputPath2.getAbsolutePath()));
+      union.write(To.avroFile(outputPath1));
+      pipeline.write(union, To.avroFile(outputPath2));
 
       pipeline.run();
 
-      checkFileContents(outputPath1.getAbsolutePath());
-      checkFileContents(outputPath2.getAbsolutePath());
+      checkFileContents(outputPath1);
+      checkFileContents(outputPath2);
 
     } else {
 
-      union.write(To.textFile(outputPath1.getAbsolutePath()));
-      pipeline.write(union, To.textFile(outputPath2.getAbsolutePath()));
-      pipeline.writeTextFile(union, outputPath3.getAbsolutePath());
+      union.write(To.textFile(outputPath1));
+      pipeline.write(union, To.textFile(outputPath2));
+      pipeline.writeTextFile(union, outputPath3);
 
       pipeline.run();
 
-      checkFileContents(outputPath1.getAbsolutePath());
-      checkFileContents(outputPath2.getAbsolutePath());
-      checkFileContents(outputPath3.getAbsolutePath());
+      checkFileContents(outputPath1);
+      checkFileContents(outputPath2);
+      checkFileContents(outputPath3);
     }
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java b/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java
index c64b8ec..4f65964 100644
--- a/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java
+++ b/crunch/src/it/java/org/apache/crunch/lib/CogroupIT.java
@@ -36,17 +36,20 @@ import org.apache.crunch.fn.MapKeysFn;
 import org.apache.crunch.fn.MapValuesFn;
 import org.apache.crunch.impl.mr.MRPipeline;
 import org.apache.crunch.io.From;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTableType;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.avro.AvroTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.base.Splitter;
 import com.google.common.io.Files;
 
 public class CogroupIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   private static class WordSplit extends DoFn<String, Pair<String, Long>> {
     @Override
@@ -99,9 +102,9 @@ public class CogroupIT {
   }
 
   public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
-    String shakesInputPath = FileHelper.createTempCopyOf("shakes.txt");
-    String maughamInputPath = FileHelper.createTempCopyOf("maugham.txt");
-    File output = FileHelper.createOutputPath();
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
+    String maughamInputPath = tmpDir.copyResourceFileName("maugham.txt");
+    File output = tmpDir.getFile("output");
 
     PCollection<String> shakespeare = pipeline.read(From.textFile(shakesInputPath));
     PCollection<String> maugham = pipeline.read(From.textFile(maughamInputPath));
@@ -118,7 +121,5 @@ public class CogroupIT {
       }
     }
     assertTrue(passed);
-
-    output.deleteOnExit();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
new file mode 100644
index 0000000..9218213
--- /dev/null
+++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
@@ -0,0 +1,105 @@
+package org.apache.crunch.test;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+
+
+/**
+ * Creates a temporary directory for a test case and destroys it 
+ * afterwards. This provides a temporary directory like JUnit's 
+ * {@link TemporaryFolder} but geared towards Hadoop applications.
+ * Unlike {@link TemporaryFolder}, it doesn't create any files or
+ * directories except for the root directory itself.
+ */
+public final class TemporaryPath extends ExternalResource {
+  private TemporaryFolder tmp = new TemporaryFolder();
+  
+  @Override
+  public Statement apply(Statement base, Description description) {
+    return tmp.apply(base, description);
+  }
+  
+  /**
+   * Get the root directory which will be deleted automatically.
+   */
+  public File getRootFile() {
+    return tmp.getRoot();
+  }
+  
+  /**
+   * Get the root directory as a {@link Path}.
+   */
+  public Path getRootPath() {
+    return toPath(tmp.getRoot());
+  }
+  
+  /**
+   * Get the root directory as an absolute file name.
+   */
+  public String getRootFileName() {
+    return tmp.getRoot().getAbsolutePath();
+  }
+  
+  /**
+   * Get a {@link File} below the temporary directory.
+   */
+  public File getFile(String fileName) {
+    return new File(getRootFile(), fileName);
+  }
+  
+  /**
+   * Get a {@link Path} below the temporary directory.
+   */
+  public Path getPath(String fileName) {
+    return toPath(getFile(fileName));
+  }
+  
+  /**
+   * Get an absolute file name below the temporary directory.
+   */
+  public String getFileName(String fileName) {
+    return getFile(fileName).getAbsolutePath();
+  }
+  
+  /**
+   * Copy a classpath resource to {@link File}.
+   */
+  public File copyResourceFile(String resourceName) throws IOException {
+    File dest = new File(tmp.getRoot(), resourceName);
+    copy(resourceName, dest);
+    return dest;
+  }
+
+  /**
+   * Copy a classpath resource to a {@link Path}
+   */
+  public Path copyResourcePath(String resourceName) throws IOException {
+    return toPath(copyResourceFile(resourceName));
+  }
+
+  /**
+   * Copy a classpath resource returning its absolute file name.
+   */
+  public String copyResourceFileName(String resourceName) throws IOException {
+    return copyResourceFile(resourceName).getAbsolutePath();
+  }
+  
+  private void copy(String resourceName, File dest) throws IOException {
+    Files.copy(
+        Resources.newInputStreamSupplier(Resources.getResource(resourceName)),
+        dest);
+  }
+  
+  private static Path toPath(File file) {
+    return new Path(file.getAbsolutePath());
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6f549ed3/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java
new file mode 100644
index 0000000..65018da
--- /dev/null
+++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java
@@ -0,0 +1,59 @@
+package org.apache.crunch.test;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.junit.Rule;
+import org.junit.Test;
+
+
+public class TemporaryPathIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
+  
+  @Test
+  public void testRoot() throws IOException {
+    assertThat(tmpDir.getRootFile().exists(), is(true));
+    assertThat(new File(tmpDir.getRootFileName()).exists(), is(true));
+    assertThat(getFs().exists(tmpDir.getRootPath()), is(true));
+  }
+
+  @Test
+  public void testFile() throws IOException {
+    assertThat(tmpDir.getFile("foo").getParentFile(), is(tmpDir.getRootFile()));
+    assertThat(tmpDir.getFile("foo").getName(), is("foo"));
+    assertThat(tmpDir.getFile("foo").exists(), is(false));
+  }
+
+  @Test
+  public void testPath() throws IOException {
+    assertThat(tmpDir.getPath("foo").getParent(), is(tmpDir.getRootPath()));
+    assertThat(tmpDir.getPath("foo").getName(), is("foo"));
+    assertThat(getFs().exists(tmpDir.getPath("foo")), is(false));
+  }
+  
+  @Test
+  public void testFileName() {
+    assertThat(new File(tmpDir.getRootFileName()), is(tmpDir.getRootFile()));
+    assertThat(new File(tmpDir.getFileName("foo").toString()), is(tmpDir.getFile("foo")));
+  }
+  
+  @Test
+  public void testCopyResource() throws IOException {
+    File dest = tmpDir.getFile("shakes.txt");
+    assertThat(dest.exists(), is(false));
+    
+    tmpDir.copyResourceFile("shakes.txt");
+    assertThat(dest.exists(), is(true));
+  }
+
+  private LocalFileSystem getFs() throws IOException {
+    return FileSystem.getLocal(new Configuration());
+  }
+}


Mime
View raw message