crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [01/43] CRUNCH-196: crunch -> crunch-core rename to fix build issues
Date Tue, 23 Apr 2013 20:41:03 GMT
Updated Branches:
  refs/heads/master cbc7c2fb3 -> 890e0086a


http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
----------------------------------------------------------------------
diff --git a/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java b/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
deleted file mode 100644
index 5396fba..0000000
--- a/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.crunch.types.writable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.crunch.Pair;
-import org.apache.crunch.Tuple3;
-import org.apache.crunch.Tuple4;
-import org.apache.crunch.TupleN;
-import org.apache.crunch.types.PTableType;
-import org.apache.crunch.types.PType;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-public class WritablesTest {
-
-  @Test
-  public void testNulls() throws Exception {
-    Void n = null;
-    NullWritable nw = NullWritable.get();
-    testInputOutputFn(Writables.nulls(), n, nw);
-  }
-
-  @Test
-  public void testStrings() throws Exception {
-    String s = "abc";
-    Text text = new Text(s);
-    testInputOutputFn(Writables.strings(), s, text);
-  }
-
-  @Test
-  public void testInts() throws Exception {
-    int j = 55;
-    IntWritable w = new IntWritable(j);
-    testInputOutputFn(Writables.ints(), j, w);
-  }
-
-  @Test
-  public void testLongs() throws Exception {
-    long j = 55;
-    LongWritable w = new LongWritable(j);
-    testInputOutputFn(Writables.longs(), j, w);
-  }
-
-  @Test
-  public void testFloats() throws Exception {
-    float j = 55.5f;
-    FloatWritable w = new FloatWritable(j);
-    testInputOutputFn(Writables.floats(), j, w);
-  }
-
-  @Test
-  public void testDoubles() throws Exception {
-    double j = 55.5d;
-    DoubleWritable w = new DoubleWritable(j);
-    testInputOutputFn(Writables.doubles(), j, w);
-  }
-
-  @Test
-  public void testBoolean() throws Exception {
-    boolean j = false;
-    BooleanWritable w = new BooleanWritable(j);
-    testInputOutputFn(Writables.booleans(), j, w);
-  }
-
-  @Test
-  public void testBytes() throws Exception {
-    byte[] bytes = new byte[] { 17, 26, -98 };
-    BytesWritable bw = new BytesWritable(bytes);
-    ByteBuffer bb = ByteBuffer.wrap(bytes);
-    testInputOutputFn(Writables.bytes(), bb, bw);
-  }
-
-  @Test
-  public void testCollections() throws Exception {
-    String s = "abc";
-    Collection<String> j = Lists.newArrayList();
-    j.add(s);
-    GenericArrayWritable<Text> w = new GenericArrayWritable<Text>(Text.class);
-    w.set(new Text[] { new Text(s) });
-    testInputOutputFn(Writables.collections(Writables.strings()), j, w);
-  }
-
-  @Test
-  public void testPairs() throws Exception {
-    Pair<String, String> j = Pair.of("a", "b");
-    TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), });
-    w.setWritten(0);
-    w.setWritten(1);
-    testInputOutputFn(Writables.pairs(Writables.strings(), Writables.strings()), j, w);
-  }
-
-  @Test
-  public void testNestedTables() throws Exception {
-    PTableType<Long, Long> pll = Writables.tableOf(Writables.longs(), Writables.longs());
-    PTableType<Pair<Long, Long>, String> nest = Writables.tableOf(pll, Writables.strings());
-    assertNotNull(nest);
-  }
-
-  @Test
-  public void testPairEquals() throws Exception {
-    PType<Pair<Long, ByteBuffer>> t1 = Writables.pairs(Writables.longs(), Writables.bytes());
-    PType<Pair<Long, ByteBuffer>> t2 = Writables.pairs(Writables.longs(), Writables.bytes());
-    assertEquals(t1, t2);
-    assertEquals(t1.hashCode(), t2.hashCode());
-  }
-
-  @Test
-  @SuppressWarnings("rawtypes")
-  public void testTriples() throws Exception {
-    Tuple3 j = Tuple3.of("a", "b", "c");
-    TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), new Text("c"),
});
-    w.setWritten(0);
-    w.setWritten(1);
-    w.setWritten(2);
-    WritableType<?, ?> wt = Writables.triples(Writables.strings(), Writables.strings(),
Writables.strings());
-    testInputOutputFn(wt, j, w);
-  }
-
-  @Test
-  @SuppressWarnings("rawtypes")
-  public void testQuads() throws Exception {
-    Tuple4 j = Tuple4.of("a", "b", "c", "d");
-    TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), new Text("c"),
new Text("d"), });
-    w.setWritten(0);
-    w.setWritten(1);
-    w.setWritten(2);
-    w.setWritten(3);
-    WritableType<?, ?> wt = Writables.quads(Writables.strings(), Writables.strings(),
Writables.strings(),
-        Writables.strings());
-    testInputOutputFn(wt, j, w);
-  }
-
-  @Test
-  public void testTupleN() throws Exception {
-    TupleN j = new TupleN("a", "b", "c", "d", "e");
-    TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), new Text("c"),
new Text("d"),
-        new Text("e"), });
-    w.setWritten(0);
-    w.setWritten(1);
-    w.setWritten(2);
-    w.setWritten(3);
-    w.setWritten(4);
-    WritableType<?, ?> wt = Writables.tuples(Writables.strings(), Writables.strings(),
Writables.strings(),
-        Writables.strings(), Writables.strings());
-    testInputOutputFn(wt, j, w);
-  }
-
-  protected static class TestWritable implements Writable {
-    String left;
-    int right;
-
-    @Override
-    public void write(DataOutput out) throws IOException {
-      out.writeUTF(left);
-      out.writeInt(right);
-    }
-
-    @Override
-    public void readFields(DataInput in) throws IOException {
-      left = in.readUTF();
-      right = in.readInt();
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-      if (this == obj)
-        return true;
-      if (obj == null)
-        return false;
-      if (getClass() != obj.getClass())
-        return false;
-      TestWritable other = (TestWritable) obj;
-      if (left == null) {
-        if (other.left != null)
-          return false;
-      } else if (!left.equals(other.left))
-        return false;
-      if (right != other.right)
-        return false;
-      return true;
-    }
-
-  }
-
-  @Test
-  public void testRecords() throws Exception {
-    TestWritable j = new TestWritable();
-    j.left = "a";
-    j.right = 1;
-    TestWritable w = new TestWritable();
-    w.left = "a";
-    w.right = 1;
-    WritableType<?, ?> wt = Writables.records(TestWritable.class);
-    testInputOutputFn(wt, j, w);
-  }
-
-  @Test
-  public void testTableOf() throws Exception {
-    Pair<String, String> j = Pair.of("a", "b");
-    Pair<Text, Text> w = Pair.of(new Text("a"), new Text("b"));
-    WritableTableType<String, String> wtt = Writables.tableOf(Writables.strings(),
Writables.strings());
-    testInputOutputFn(wtt, j, w);
-  }
-
-  @Test
-  public void testRegister() throws Exception {
-    WritableType<TestWritable, TestWritable> wt = Writables.writables(TestWritable.class);
-    Writables.register(TestWritable.class, wt);
-    assertSame(Writables.records(TestWritable.class), wt);
-  }
-
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  protected static void testInputOutputFn(PType ptype, Object java, Object writable) {
-    ptype.getInputMapFn().initialize();
-    ptype.getOutputMapFn().initialize();
-    assertEquals(java, ptype.getInputMapFn().map(writable));
-    assertEquals(writable, ptype.getOutputMapFn().map(java));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java
----------------------------------------------------------------------
diff --git a/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java b/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java
deleted file mode 100644
index 6784f14..0000000
--- a/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.crunch.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class DistCacheTest {
-
-  // A temporary folder used to hold files created for the test.
-  @Rule
-  public TemporaryFolder testFolder = new TemporaryFolder();
-
-  // A configuration and lists of paths to use in tests.
-  private Configuration testConf;
-  private String[] testFilePaths;
-  private String[] testFileQualifiedPaths;
-
-  /**
-   * Setup resources for tests. These include:
-   * <ol>
-   * <li>A Hadoop configuration.
-   * <li>A directory of temporary files that includes 3 .jar files and 1 other
-   * file.
-   * <li>Arrays containing the canonical paths and qualified paths to the test
-   * files.
-   * </ol>
-   */
-  @Before
-  public void setup() throws IOException {
-    // Create a configuration for tests.
-    testConf = new Configuration();
-
-    // Create the test files and add their paths to the list of test file paths.
-    testFilePaths = new String[3];
-    testFilePaths[0] = testFolder.newFile("jar1.jar").getCanonicalPath();
-    testFilePaths[1] = testFolder.newFile("jar2.jar").getCanonicalPath();
-    testFilePaths[2] = testFolder.newFile("jar3.jar").getCanonicalPath();
-    testFolder.newFile("notJar.other");
-
-    // Populate a list of qualified paths from the test file paths.
-    testFileQualifiedPaths = new String[3];
-    for (int i = 0; i < testFilePaths.length; i++) {
-      testFileQualifiedPaths[i] = "file:" + testFilePaths[i];
-    }
-  }
-
-  /**
-   * Tests adding jars one-by-one to a job's configuration.
-   * 
-   * @throws IOException
-   *           If there is a problem adding the jars.
-   */
-  @Test
-  public void testAddJar() throws IOException {
-    // Add each valid jar path to the distributed cache configuration, and
-    // verify each was
-    // added correctly in turn.
-    for (int i = 0; i < testFilePaths.length; i++) {
-      DistCache.addJarToDistributedCache(testConf, testFilePaths[i]);
-      assertEquals("tmpjars configuration var does not contain expected value.",
-          StringUtils.join(testFileQualifiedPaths, ",", 0, i + 1), testConf.get("tmpjars"));
-    }
-  }
-
-  /**
-   * Tests that attempting to add the path to a jar that does not exist to the
-   * configuration throws an exception.
-   * 
-   * @throws IOException
-   *           If the added jar path does not exist. This exception is expected.
-   */
-  @Test(expected = IOException.class)
-  public void testAddJarThatDoesntExist() throws IOException {
-    DistCache.addJarToDistributedCache(testConf, "/garbage/doesntexist.jar");
-  }
-
-  /**
-   * Tests that adding a directory of jars to the configuration works as
-   * expected. .jar files under the added directory should be added to the
-   * configuration, and all other files should be skipped.
-   * 
-   * @throws IOException
-   *           If there is a problem adding the jar directory to the
-   *           configuration.
-   */
-  @Test
-  public void testAddJarDirectory() throws IOException {
-    DistCache.addJarDirToDistributedCache(testConf, testFolder.getRoot().getCanonicalPath());
-    // Throw the added jar paths in a set to detect duplicates.
-    String[] splitJarPaths = StringUtils.split(testConf.get("tmpjars"), ",");
-    Set<String> addedJarPaths = new HashSet<String>();
-    for (String path : splitJarPaths) {
-      addedJarPaths.add(path);
-    }
-    assertEquals("Incorrect number of jar paths added.", testFilePaths.length, addedJarPaths.size());
-
-    // Ensure all expected paths were added.
-    for (int i = 0; i < testFileQualifiedPaths.length; i++) {
-      assertTrue("Expected jar path missing from jar paths added to tmpjars: " + testFileQualifiedPaths[i],
-          addedJarPaths.contains(testFileQualifiedPaths[i]));
-    }
-  }
-
-  /**
-   * Tests that adding a jar directory that does not exist to the configuration
-   * throws an exception.
-   * 
-   * @throws IOException
-   *           If the added jar directory does not exist. This exception is
-   *           expected.
-   */
-  @Test(expected = IOException.class)
-  public void testAddJarDirectoryThatDoesntExist() throws IOException {
-    DistCache.addJarDirToDistributedCache(testConf, "/garbage/doesntexist");
-  }
-
-  /**
-   * Tests that adding a jar directory that is not a directory to the
-   * configuration throws an exception.
-   * 
-   * @throws IOException
-   *           If the added jar directory is not a directory. This exception is
-   *           expected.
-   */
-  @Test(expected = IOException.class)
-  public void testAddJarDirectoryNotDirectory() throws IOException {
-    DistCache.addJarDirToDistributedCache(testConf, testFilePaths[0]);
-  }
-}

http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d5f90f2..71f5e0f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,7 +44,7 @@ under the License.
   </prerequisites>
 
   <modules>
-    <module>crunch</module>
+    <module>crunch-core</module>
     <module>crunch-hbase</module>
     <module>crunch-test</module>
     <module>crunch-contrib</module>
@@ -103,7 +103,7 @@ under the License.
     <dependencies>
       <dependency>
         <groupId>org.apache.crunch</groupId>
-        <artifactId>crunch</artifactId>
+        <artifactId>crunch-core</artifactId>
         <version>${project.version}</version>
       </dependency>
 


Mime
View raw message