incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [2/4] git commit: CRUNCH-24: Clean up data after integration test.
Date Thu, 26 Jul 2012 05:48:26 GMT
CRUNCH-24: Clean up data after integration test.

Redirect more test output to self-cleaning temporary directories.

Signed-off-by: jwills <jwills@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/e540459e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/e540459e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/e540459e

Branch: refs/heads/master
Commit: e540459ee2f4c3b7f29989c16cb9aa19278fd3fb
Parents: be357f2
Author: Matthias Friedrich <matt@mafr.de>
Authored: Wed Jul 25 18:58:44 2012 +0200
Committer: jwills <jwills@apache.org>
Committed: Wed Jul 25 22:23:38 2012 -0700

----------------------------------------------------------------------
 .../it/java/org/apache/crunch/CollectionsIT.java   |    2 -
 crunch/src/it/java/org/apache/crunch/MapsIT.java   |   19 ++++++--------
 .../java/org/apache/crunch/WordCountHBaseIT.java   |    3 +-
 .../crunch/impl/mem/MemPipelineFileWritingIT.java  |    8 ++++-
 .../apache/crunch/io/CompositePathIterableIT.java  |   16 +++++++-----
 .../java/org/apache/crunch/test/TemporaryPath.java |    3 +-
 6 files changed, 27 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/CollectionsIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/CollectionsIT.java b/crunch/src/it/java/org/apache/crunch/CollectionsIT.java
index e6be525..c9ff2a9 100644
--- a/crunch/src/it/java/org/apache/crunch/CollectionsIT.java
+++ b/crunch/src/it/java/org/apache/crunch/CollectionsIT.java
@@ -24,7 +24,6 @@ import java.util.Collection;
 
 import org.apache.crunch.impl.mem.MemPipeline;
 import org.apache.crunch.impl.mr.MRPipeline;
-import org.apache.crunch.impl.mr.run.RuntimeParameters;
 import org.apache.crunch.test.FileHelper;
 import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
@@ -33,7 +32,6 @@ import org.apache.crunch.types.writable.WritableTypeFamily;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/MapsIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/MapsIT.java b/crunch/src/it/java/org/apache/crunch/MapsIT.java
index e0911c9..3b9fb37 100644
--- a/crunch/src/it/java/org/apache/crunch/MapsIT.java
+++ b/crunch/src/it/java/org/apache/crunch/MapsIT.java
@@ -17,41 +17,38 @@
  */
 package org.apache.crunch;
 
+import java.io.Serializable;
 import java.util.Map;
 
 import org.apache.crunch.impl.mr.MRPipeline;
-import org.apache.crunch.impl.mr.run.RuntimeParameters;
-import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.PTypeFamily;
 import org.apache.crunch.types.avro.AvroTypeFamily;
 import org.apache.crunch.types.writable.WritableTypeFamily;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 
 public class MapsIT {
   @Rule
-  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   @Test
   public void testWritables() throws Exception {
-    run(WritableTypeFamily.getInstance(), temporaryFolder.newFolder().getAbsolutePath());
+    run(WritableTypeFamily.getInstance(), tmpDir);
   }
 
   @Test
   public void testAvros() throws Exception {
-    run(AvroTypeFamily.getInstance(), temporaryFolder.newFolder().getAbsolutePath());
+    run(AvroTypeFamily.getInstance(), tmpDir);
   }
 
-  public static void run(PTypeFamily typeFamily, String path) throws Exception {
-    Configuration config = new Configuration();
-    config.set(RuntimeParameters.TMP_DIR, path);
-    Pipeline pipeline = new MRPipeline(MapsIT.class, config);
-    String shakesInputPath = FileHelper.createTempCopyOf("shakes.txt");
+  public static void run(PTypeFamily typeFamily, TemporaryPath tmpDir) throws Exception {
+    Pipeline pipeline = new MRPipeline(MapsIT.class, tmpDir.setTempLoc(new Configuration()));
+    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
     PCollection<String> shakespeare = pipeline.readTextFile(shakesInputPath);
     Iterable<Pair<String, Map<String, Long>>> output = shakespeare
         .parallelDo(new DoFn<String, Pair<String, Map<String, Long>>>()
{

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
index 9704c3b..9d42ee0 100644
--- a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
+++ b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java
@@ -92,7 +92,8 @@ public class WordCountHBaseIT {
   @Before
   public void setUp() throws Exception {
     Configuration conf = hbaseTestUtil.getConfiguration();
-    conf.set("hadoop.log.dir", tmpDir.getRootFileName());
+    conf.set("hadoop.log.dir", tmpDir.getFileName("logs"));
+    conf.set("hadoop.tmp.dir", tmpDir.getFileName("hadoop-tmp"));
     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
     conf.setInt("hbase.master.info.port", -1);
     conf.setInt("hbase.regionserver.info.port", -1);

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java b/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java
index f4a0a90..db0e8ce 100644
--- a/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java
+++ b/crunch/src/it/java/org/apache/crunch/impl/mem/MemPipelineFileWritingIT.java
@@ -25,6 +25,8 @@ import java.util.List;
 
 import org.apache.crunch.PCollection;
 import org.apache.crunch.Pipeline;
+import org.apache.crunch.test.TemporaryPath;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.base.Charsets;
@@ -32,10 +34,12 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.io.Files;
 
 public class MemPipelineFileWritingIT {
+  @Rule
+  public TemporaryPath baseTmpDir = new TemporaryPath();
+
   @Test
   public void testMemPipelineFileWriter() throws Exception {
-    File tmpDir = Files.createTempDir();
-    tmpDir.delete();
+    File tmpDir = baseTmpDir.getRootFile();
     Pipeline p = MemPipeline.getInstance();
     PCollection<String> lines = MemPipeline.collectionOf("hello", "world");
     p.writeTextFile(lines, tmpDir.getAbsolutePath());

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java b/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
index 0f1a45e..c55287c 100644
--- a/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
+++ b/crunch/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java
@@ -26,17 +26,20 @@ import java.io.IOException;
 
 import org.apache.crunch.io.text.TextFileReaderFactory;
 import org.apache.crunch.test.FileHelper;
+import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.types.writable.Writables;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
-import com.google.common.io.Files;
 
 public class CompositePathIterableIT {
+  @Rule
+  public TemporaryPath tmpDir = new TemporaryPath();
 
   @Test
   public void testCreate_FilePresent() throws IOException {
@@ -53,12 +56,12 @@ public class CompositePathIterableIT {
 
   @Test
   public void testCreate_DirectoryPresentButNoFiles() throws IOException {
-    String inputFilePath = Files.createTempDir().getAbsolutePath();
+    Path emptyInputDir = tmpDir.getRootPath();
 
     Configuration conf = new Configuration();
     LocalFileSystem local = FileSystem.getLocal(conf);
 
-    Iterable<String> iterable = CompositePathIterable.create(local, new Path(inputFilePath),
+    Iterable<String> iterable = CompositePathIterable.create(local, emptyInputDir,
         new TextFileReaderFactory<String>(Writables.strings(), conf));
 
     assertTrue(Lists.newArrayList(iterable).isEmpty());
@@ -66,16 +69,15 @@ public class CompositePathIterableIT {
 
   @Test(expected = IOException.class)
   public void testCreate_DirectoryNotPresent() throws IOException {
-    File inputFileDir = Files.createTempDir();
-    inputFileDir.delete();
+    File nonExistentDir = tmpDir.getFile("not-there");
 
     // Sanity check
-    assertFalse(inputFileDir.exists());
+    assertFalse(nonExistentDir.exists());
 
     Configuration conf = new Configuration();
     LocalFileSystem local = FileSystem.getLocal(conf);
 
-    CompositePathIterable.create(local, new Path(inputFileDir.getAbsolutePath()), new TextFileReaderFactory<String>(
+    CompositePathIterable.create(local, new Path(nonExistentDir.getAbsolutePath()), new TextFileReaderFactory<String>(
         Writables.strings(), conf));
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e540459e/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
index 7bdc91a..dd4f505 100644
--- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
+++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
@@ -103,7 +103,8 @@ public final class TemporaryPath extends ExternalResource {
   }
 
   public Configuration setTempLoc(Configuration config) throws IOException {
-    config.set(RuntimeParameters.TMP_DIR, tmp.newFolder().getAbsolutePath());
+    config.set(RuntimeParameters.TMP_DIR, getRootFileName());
+    config.set("hadoop.tmp.dir", getFileName("hadoop-tmp"));
     return config;
   }
 }


Mime
View raw message