crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gr...@apache.org
Subject crunch git commit: CRUNCH-524 Correctly delete temp files
Date Sat, 23 May 2015 19:34:33 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 9bb526fd1 -> 6c8c5ffd0


CRUNCH-524 Correctly delete temp files

Delete /tmp/crunch- on exit; delete ~/hbase after test finishes; move H2 db
file to /tmp

Signed-off-by: Gabriel Reid <greid@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6c8c5ffd
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6c8c5ffd
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6c8c5ffd

Branch: refs/heads/master
Commit: 6c8c5ffd0447da82e63cf0b74ed2026c77a0c554
Parents: 9bb526f
Author: Sean Owen <sowen@cloudera.com>
Authored: Thu May 21 22:34:16 2015 +0100
Committer: Gabriel Reid <greid@apache.org>
Committed: Sat May 23 21:34:04 2015 +0200

----------------------------------------------------------------------
 .../crunch/contrib/io/jdbc/DataBaseSourceIT.java  |  5 +++--
 .../crunch/impl/dist/DistributedPipeline.java     |  4 +++-
 .../apache/crunch/io/hbase/WordCountHBaseIT.java  | 18 ++++++++++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
----------------------------------------------------------------------
diff --git a/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
b/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
index 1c48559..a19803e 100644
--- a/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
+++ b/crunch-contrib/src/it/java/org/apache/crunch/contrib/io/jdbc/DataBaseSourceIT.java
@@ -46,12 +46,13 @@ public class DataBaseSourceIT extends CrunchTestSupport implements Serializable
   public void start() throws Exception {
     File file = tempDir.copyResourceFile("data.script");
     server = Server.createTcpServer().start();
-    RunScript.execute("jdbc:h2:tcp://localhost/~/test", "sa", "", file.getAbsolutePath(),
"utf-8", false);
+    RunScript.execute("jdbc:h2:file:/tmp/test", "sa", "", file.getAbsolutePath(), "utf-8",
false);
   }
 
   @After
   public void stop() throws Exception {
     server.stop();
+    new File("/tmp/test.h2.db").delete();
   }
 
   @Test
@@ -59,7 +60,7 @@ public class DataBaseSourceIT extends CrunchTestSupport implements Serializable
     Pipeline pipeline = new MRPipeline(DataBaseSourceIT.class);
     DataBaseSource<IdentifiableName> dbsrc = new DataBaseSource.Builder<IdentifiableName>(IdentifiableName.class)
         .setDriverClass(org.h2.Driver.class)
-        .setUrl("jdbc:h2:tcp://localhost/~/test").setUsername("sa").setPassword("")
+        .setUrl("jdbc:h2:file:/tmp/test").setUsername("sa").setPassword("")
         .selectSQLQuery("SELECT ID, NAME FROM TEST").countSQLQuery("select count(*) from
Test").build();
 
     PCollection<IdentifiableName> cdidata = pipeline.read(dbsrc);

http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
b/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
index 88da5a6..049046a 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/dist/DistributedPipeline.java
@@ -396,7 +396,9 @@ public abstract class DistributedPipeline implements Pipeline {
   private static Path createTempDirectory(Configuration conf) {
     Path dir = createTemporaryPath(conf);
     try {
-      dir.getFileSystem(conf).mkdirs(dir);
+      FileSystem fs = dir.getFileSystem(conf);
+      fs.mkdirs(dir);
+      fs.deleteOnExit(dir);
     } catch (IOException e) {
       throw new RuntimeException("Cannot create job output directory " + dir, e);
     }

http://git-wip-us.apache.org/repos/asf/crunch/blob/6c8c5ffd/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
index dd48352..ead2aea 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/WordCountHBaseIT.java
@@ -19,6 +19,7 @@ package org.apache.crunch.io.hbase;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -39,6 +40,8 @@ import org.apache.crunch.test.TemporaryPath;
 import org.apache.crunch.test.TemporaryPaths;
 import org.apache.crunch.types.writable.Writables;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.client.Delete;
@@ -119,6 +122,18 @@ public class WordCountHBaseIT {
   @Before
   public void setUp() throws Exception {
     Configuration conf = HBaseConfiguration.create(tmpDir.getDefaultConfiguration());
+
+    // HBase 0.98 HBaseTestingUtility hard-codes ~/hbase as its temp directory and leaves
+    // it lying around after this test. It's easy to delete, but to be sure it's safe to
+    // delete, we need to be sure that it doesn't exist already. The test probably won't
+    // work anyway if it does. If it doesn't exist, then deleting it later doesn't matter
+    // even if a future HBase is used (which doesn't have this problem) and this test
+    // isn't updated.
+    FileSystem fs = FileSystem.get(conf);
+    Path hbaseTestPath = new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+    assertFalse("The hbase directory in your home directory already exists. This test deletes
this directory " +
+            "on exit, so it will not run unless you delete the directory yourself", fs.exists(hbaseTestPath));
+
     hbaseTestUtil = new HBaseTestingUtility(conf);
     hbaseTestUtil.startMiniZKCluster();
     hbaseTestUtil.startMiniHBaseCluster(1, 1);
@@ -133,6 +148,9 @@ public class WordCountHBaseIT {
   public void tearDown() throws Exception {
     hbaseTestUtil.shutdownMiniHBaseCluster();
     hbaseTestUtil.shutdownMiniZKCluster();
+    FileSystem fs = FileSystem.get(hbaseTestUtil.getConfiguration());
+    Path hbaseTestPath = new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+    fs.delete(hbaseTestPath, true);
   }
 
   public void run(Pipeline pipeline) throws Exception {


Mime
View raw message