incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Made some more adjustments to get the tests to pass.
Date Wed, 02 Jul 2014 15:39:35 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/apache-blur-0.2 a1f57f4ee -> 4147f3588


Made some more adjustments to get the tests to pass.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4147f358
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4147f358
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4147f358

Branch: refs/heads/apache-blur-0.2
Commit: 4147f358809814d5a8472b804244c697d6861b15
Parents: a1f57f4
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Jul 2 11:39:20 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Jul 2 11:39:20 2014 -0400

----------------------------------------------------------------------
 blur-mapred-hadoop1/pom.xml                     |  2 +-
 .../mapreduce/lib/BlurOutputFormatTest.java     | 39 ++++++--------------
 .../blur/mapreduce/lib/CsvBlurDriverTest.java   | 14 +++++++
 3 files changed, 27 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4147f358/blur-mapred-hadoop1/pom.xml
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/pom.xml b/blur-mapred-hadoop1/pom.xml
index 53bf1e3..bc65493 100644
--- a/blur-mapred-hadoop1/pom.xml
+++ b/blur-mapred-hadoop1/pom.xml
@@ -194,7 +194,7 @@ under the License.
 				      <groupId>org.apache.mrunit</groupId>
 				      <artifactId>mrunit</artifactId>
 				      <version>${mrunit.version}</version>
-				      <classifier>hadoop1</classifier>
+				      <classifier>hadoop2</classifier>
 					  <scope>test</scope>
                 </dependency>
 			</dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4147f358/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
index 11ad0ce..96723c8 100644
--- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
@@ -56,15 +56,14 @@ public class BlurOutputFormatTest {
   private static Configuration conf = new Configuration();
   private static FileSystem localFs;
   private static MiniMRCluster mr;
-  private static Path TEST_ROOT_DIR;
+  private static final Path TEST_ROOT_DIR = new Path("./target/tmp/BlurOutputFormatTest_tmp");
   private static JobConf jobConf;
-  private Path outDir = new Path(TEST_ROOT_DIR + "/out");
-  private Path inDir = new Path(TEST_ROOT_DIR + "/in");
+  private static final Path outDir = new Path(TEST_ROOT_DIR + "/out");
+  private static final Path inDir = new Path(TEST_ROOT_DIR + "/in");
 
   @BeforeClass
   public static void setupTest() throws Exception {
     System.setProperty("test.build.data", "./target/BlurOutputFormatTest/data");
-    TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "target/tmp/BlurOutputFormatTest_tmp"));
     System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
     try {
       localFs = FileSystem.getLocal(conf);
@@ -97,18 +96,18 @@ public class BlurOutputFormatTest {
   }
 
   @Before
-  public void setup() {
+  public void setup() throws IllegalArgumentException, IOException {
     TableContext.clear();
+    assertTrue(localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true));
+    assertTrue(localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true));
   }
 
   @Test
   public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException
{
-    localFs.delete(inDir, true);
-    localFs.delete(outDir, true);
     writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
     writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
 
-    Job job = new Job(jobConf, "blur index");
+    Job job = new Job(new Configuration(jobConf), "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
     job.setInputFormatClass(TextInputFormat.class);
@@ -153,13 +152,10 @@ public class BlurOutputFormatTest {
 
   @Test
   public void testBlurOutputFormatOverFlowTest() throws IOException, InterruptedException,
ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
     writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
     writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
 
-    Job job = new Job(jobConf, "blur index");
+    Job job = new Job(new Configuration(jobConf), "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
     job.setInputFormatClass(TextInputFormat.class);
@@ -195,13 +191,10 @@ public class BlurOutputFormatTest {
   @Test
   public void testBlurOutputFormatOverFlowMultipleReducersTest() throws IOException, InterruptedException,
       ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
     writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
     writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
 
-    Job job = new Job(jobConf, "blur index");
+    Job job = new Job(new Configuration(jobConf), "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
     job.setInputFormatClass(TextInputFormat.class);
@@ -243,13 +236,10 @@ public class BlurOutputFormatTest {
   @Test
   public void testBlurOutputFormatOverFlowMultipleReducersWithReduceMultiplierTest() throws
IOException,
       InterruptedException, ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
     writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
     writeRecordsFile("in/part2", 1, 50, 2000, 100, "cf1"); // 100 * 50 = 5,000
 
-    Job job = new Job(jobConf, "blur index");
+    Job job = new Job(new Configuration(jobConf), "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
     job.setInputFormatClass(TextInputFormat.class);
@@ -291,12 +281,10 @@ public class BlurOutputFormatTest {
   @Test(expected = IllegalArgumentException.class)
   public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException,
       ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
     writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
     writeRecordsFile("in/part2", 1, 1, 2, 1, "cf1");
 
-    Job job = new Job(jobConf, "blur index");
+    Job job = new Job(new Configuration(jobConf), "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
     job.setInputFormatClass(TextInputFormat.class);
@@ -323,14 +311,11 @@ public class BlurOutputFormatTest {
   // @Test
   public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException,
       ClassNotFoundException {
-    localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
-    localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
-
     writeRecordsFile("in/part1", 1, 50, 1, 1500, "cf1"); // 1500 * 50 = 75,000
     writeRecordsFile("in/part2", 1, 5000, 2000, 100, "cf1"); // 100 * 5000 =
                                                              // 500,000
 
-    Job job = new Job(jobConf, "blur index");
+    Job job = new Job(new Configuration(jobConf), "blur index");
     job.setJarByClass(BlurOutputFormatTest.class);
     job.setMapperClass(CsvBlurMapper.class);
     job.setInputFormatClass(TextInputFormat.class);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4147f358/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
index ec3239e..6a673eb 100644
--- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
+++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 
+import java.io.IOException;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
@@ -31,8 +32,11 @@ import org.apache.blur.mapreduce.lib.CsvBlurDriver.ControllerPool;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.compress.SnappyCodec;
 import org.apache.hadoop.mapreduce.Job;
+import org.junit.Before;
 import org.junit.Test;
 
 public class CsvBlurDriverTest {
@@ -40,6 +44,16 @@ public class CsvBlurDriverTest {
   protected String tableUri = "file:///tmp/tmppath";
   protected int shardCount = 13;
 
+  @Before
+  public void setup() throws IOException {
+    Configuration configuration = new Configuration();
+    Path path1 = new Path("file:///tmp/test1");
+    Path path2 = new Path("file:///tmp/test2");
+    FileSystem fileSystem = path1.getFileSystem(configuration);
+    fileSystem.mkdirs(path1);
+    fileSystem.mkdirs(path2);
+  }
+
   @Test
   public void testCsvBlurDriverTestFail1() throws Exception {
     Configuration configuration = new Configuration();


Mime
View raw message