incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [4/7] CRUNCH-26: Create crunch-test Maven module.
Date Sat, 28 Jul 2012 19:40:20 GMT
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java b/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java
index 24427bd..e9b2fa2 100644
--- a/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java
+++ b/crunch/src/it/java/org/apache/crunch/lib/join/MultiAvroSchemaJoinIT.java
@@ -38,7 +38,7 @@ import org.apache.crunch.io.From;
 import org.apache.crunch.test.Employee;
 import org.apache.crunch.test.Person;
 import org.apache.crunch.test.TemporaryPath;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.crunch.test.TemporaryPaths;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -52,7 +52,7 @@ public class MultiAvroSchemaJoinIT {
   private File personFile;
   private File employeeFile;
   @Rule
-  public TemporaryPath temporaryPath= new TemporaryPath();
+  public TemporaryPath tmpDir = TemporaryPaths.create();
 
   @Before
   public void setUp() throws Exception {
@@ -107,7 +107,7 @@ public class MultiAvroSchemaJoinIT {
 
   @Test
   public void testJoin() throws Exception {
-    Pipeline p = new MRPipeline(MultiAvroSchemaJoinIT.class, temporaryPath.setTempLoc(new
Configuration()));
+    Pipeline p = new MRPipeline(MultiAvroSchemaJoinIT.class, tmpDir.getDefaultConfiguration());
     PCollection<Person> people = p.read(From.avroFile(personFile.getAbsolutePath(),
records(Person.class)));
     PCollection<Employee> employees = p.read(From.avroFile(employeeFile.getAbsolutePath(),
records(Employee.class)));
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
deleted file mode 100644
index e96e3b1..0000000
--- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
+++ /dev/null
@@ -1,113 +0,0 @@
-package org.apache.crunch.test;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.crunch.impl.mr.run.RuntimeParameters;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.rules.ExternalResource;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-import com.google.common.io.Files;
-import com.google.common.io.Resources;
-
-/**
- * Creates a temporary directory for a test case and destroys it afterwards.
- * This provides a temporary directory like JUnit's {@link TemporaryFolder} but
- * geared towards Hadoop applications. Unlike {@link TemporaryFolder}, it
- * doesn't create any files or directories except for the root directory itself.
- */
-public final class TemporaryPath extends ExternalResource {
-  private TemporaryFolder tmp = new TemporaryFolder();
-
-  @Override
-  public Statement apply(Statement base, Description description) {
-    return tmp.apply(base, description);
-  }
-
-  /**
-   * Get the root directory which will be deleted automatically.
-   */
-  public File getRootFile() {
-    return tmp.getRoot();
-  }
-
-  /**
-   * Get the root directory as a {@link Path}.
-   */
-  public Path getRootPath() {
-    return toPath(tmp.getRoot());
-  }
-
-  /**
-   * Get the root directory as an absolute file name.
-   */
-  public String getRootFileName() {
-    return tmp.getRoot().getAbsolutePath();
-  }
-
-  /**
-   * Get a {@link File} below the temporary directory.
-   */
-  public File getFile(String fileName) {
-    return new File(getRootFile(), fileName);
-  }
-
-  /**
-   * Get a {@link Path} below the temporary directory.
-   */
-  public Path getPath(String fileName) {
-    return toPath(getFile(fileName));
-  }
-
-  /**
-   * Get an absolute file name below the temporary directory.
-   */
-  public String getFileName(String fileName) {
-    return getFile(fileName).getAbsolutePath();
-  }
-
-  /**
-   * Copy a classpath resource to {@link File}.
-   */
-  public File copyResourceFile(String resourceName) throws IOException {
-    File dest = new File(tmp.getRoot(), resourceName);
-    copy(resourceName, dest);
-    return dest;
-  }
-
-  /**
-   * Copy a classpath resource to a {@link Path}
-   */
-  public Path copyResourcePath(String resourceName) throws IOException {
-    return toPath(copyResourceFile(resourceName));
-  }
-
-  /**
-   * Copy a classpath resource returning its absolute file name.
-   */
-  public String copyResourceFileName(String resourceName) throws IOException {
-    return copyResourceFile(resourceName).getAbsolutePath();
-  }
-
-  private void copy(String resourceName, File dest) throws IOException {
-    Files.copy(Resources.newInputStreamSupplier(Resources.getResource(resourceName)), dest);
-  }
-
-  private static Path toPath(File file) {
-    return new Path(file.getAbsolutePath());
-  }
-
-  public Configuration getDefaultConfiguration() throws IOException {
-    return setTempLoc(new Configuration());
-  }
-  
-  public Configuration setTempLoc(Configuration config) throws IOException {
-    config.set(RuntimeParameters.TMP_DIR, getRootFileName());
-    config.set("hadoop.tmp.dir", getFileName("hadoop-tmp"));
-    return config;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java
deleted file mode 100644
index 65018da..0000000
--- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPathIT.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.apache.crunch.test;
-
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.junit.Rule;
-import org.junit.Test;
-
-
-public class TemporaryPathIT {
-  @Rule
-  public TemporaryPath tmpDir = new TemporaryPath();
-  
-  @Test
-  public void testRoot() throws IOException {
-    assertThat(tmpDir.getRootFile().exists(), is(true));
-    assertThat(new File(tmpDir.getRootFileName()).exists(), is(true));
-    assertThat(getFs().exists(tmpDir.getRootPath()), is(true));
-  }
-
-  @Test
-  public void testFile() throws IOException {
-    assertThat(tmpDir.getFile("foo").getParentFile(), is(tmpDir.getRootFile()));
-    assertThat(tmpDir.getFile("foo").getName(), is("foo"));
-    assertThat(tmpDir.getFile("foo").exists(), is(false));
-  }
-
-  @Test
-  public void testPath() throws IOException {
-    assertThat(tmpDir.getPath("foo").getParent(), is(tmpDir.getRootPath()));
-    assertThat(tmpDir.getPath("foo").getName(), is("foo"));
-    assertThat(getFs().exists(tmpDir.getPath("foo")), is(false));
-  }
-  
-  @Test
-  public void testFileName() {
-    assertThat(new File(tmpDir.getRootFileName()), is(tmpDir.getRootFile()));
-    assertThat(new File(tmpDir.getFileName("foo").toString()), is(tmpDir.getFile("foo")));
-  }
-  
-  @Test
-  public void testCopyResource() throws IOException {
-    File dest = tmpDir.getFile("shakes.txt");
-    assertThat(dest.exists(), is(false));
-    
-    tmpDir.copyResourceFile("shakes.txt");
-    assertThat(dest.exists(), is(true));
-  }
-
-  private LocalFileSystem getFs() throws IOException {
-    return FileSystem.getLocal(new Configuration());
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java
new file mode 100644
index 0000000..af8cf23
--- /dev/null
+++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPaths.java
@@ -0,0 +1,23 @@
+package org.apache.crunch.test;
+
+import org.apache.crunch.impl.mr.run.RuntimeParameters;
+import org.apache.hadoop.conf.Configuration;
+
+
+/**
+ * Utilities for working with {@link TemporaryPath}.
+ */
+public final class TemporaryPaths {
+
+  /**
+   * Static factory returning a {@link TemporaryPath} with adjusted
+   * {@link Configuration} properties.
+   */
+  public static TemporaryPath create() {
+    return new TemporaryPath(RuntimeParameters.TMP_DIR, "hadoop.tmp.dir");
+  }
+
+  private TemporaryPaths() {
+    // nothing
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/crunch/src/main/java/org/apache/crunch/test/FileHelper.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/test/FileHelper.java b/crunch/src/main/java/org/apache/crunch/test/FileHelper.java
deleted file mode 100644
index ca5a4b3..0000000
--- a/crunch/src/main/java/org/apache/crunch/test/FileHelper.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.crunch.test;
-
-import static com.google.common.io.Resources.getResource;
-import static com.google.common.io.Resources.newInputStreamSupplier;
-
-import java.io.File;
-import java.io.IOException;
-
-import com.google.common.io.Files;
-
-public class FileHelper {
-
-  public static String createTempCopyOf(String fileResource) throws IOException {
-    File tmpFile = File.createTempFile("tmp", "");
-    tmpFile.deleteOnExit();
-    Files.copy(newInputStreamSupplier(getResource(fileResource)), tmpFile);
-    return tmpFile.getAbsolutePath();
-  }
-
-  public static File createOutputPath() throws IOException {
-    File output = File.createTempFile("output", "");
-    output.delete();
-    return output;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index afa73f4..a0d7c0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,12 +34,14 @@ under the License.
 
   <modules>
     <module>crunch</module>
+    <module>crunch-test</module>
     <module>examples</module>
     <module>scrunch</module>
   </modules>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <java.source.version>1.6</java.source.version>
     <java.target.version>1.6</java.target.version>
     <scala.version>2.9.2</scala.version>
@@ -107,8 +109,7 @@ under the License.
 
       <dependency>
         <groupId>org.apache.crunch</groupId>
-        <artifactId>crunch</artifactId>
-        <type>test-jar</type>
+        <artifactId>crunch-test</artifactId>
         <version>${project.version}</version>
       </dependency>
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/scrunch/pom.xml b/scrunch/pom.xml
index 0b549ae..33e4119 100644
--- a/scrunch/pom.xml
+++ b/scrunch/pom.xml
@@ -61,19 +61,13 @@ under the License.
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
       <groupId>org.scalatest</groupId>
       <artifactId>scalatest_${scala.version}</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.crunch</groupId>
-      <artifactId>crunch</artifactId>
-      <type>test-jar</type>
+      <artifactId>crunch-test</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9c42c0b3/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
index de012c7..d418189 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
@@ -41,7 +41,7 @@ class PipelineAppTest extends ScrunchTestSupport with JUnitSuite {
     args(0) = tempDir.copyResourceFileName("shakes.txt")
     args(1) = tempDir.copyResourceFileName("maugham.txt")
     args(2) = tempDir.getFileName("output")
-    tempDir.setTempLoc(WordCount.configuration)
+    tempDir.overridePathProperties(WordCount.configuration)
     WordCount.main(args)
   }
 }


Mime
View raw message