incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/4] git commit: Moved some test output folders under target, and added some cleanup code for a hard coded "build" directory in the MR mini cluster.
Date Thu, 06 Jun 2013 21:15:12 GMT
Moved some test output folders under target, and added some cleanup code for a hard coded "build"
directory in the MR mini cluster.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/91a55795
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/91a55795
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/91a55795

Branch: refs/heads/0.1.5
Commit: 91a55795cc1afc596e5e4859efb56230529f6cb5
Parents: a5d723d
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Jun 6 17:13:46 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Jun 6 17:13:46 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/mapreduce/BlurTaskTest.java    |    2 +-
 .../blur/mapreduce/lib/BlurOutputFormatTest.java   |   58 +++++++++-----
 2 files changed, 38 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/91a55795/blur-mapred/src/test/java/org/apache/blur/mapreduce/BlurTaskTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/BlurTaskTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/BlurTaskTest.java
index e04476e..028a283 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/BlurTaskTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/BlurTaskTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
 
 @SuppressWarnings("deprecation")
 public class BlurTaskTest {
-  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
+  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp/BlurTaskTest_tmp"));
 
   @Test
   public void testGetNumReducersBadPath() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/91a55795/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
index 915ee89..937ac89 100644
--- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
+++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
 import java.io.BufferedReader;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
@@ -61,7 +62,7 @@ public class BlurOutputFormatTest {
   @BeforeClass
   public static void setup() throws Exception {
     System.setProperty("test.build.data", "./target/BlurOutputFormatTest/data");
-    TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "/tmp"));
+    TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "target/tmp/BlurOutputFormatTest_tmp"));
     System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log");
     try {
       localFs = FileSystem.getLocal(conf);
@@ -78,6 +79,19 @@ public class BlurOutputFormatTest {
     if (mr != null) {
       mr.shutdown();
     }
+    rm(new File("build"));
+  }
+
+  private static void rm(File file) {
+    if (!file.exists()) {
+      return;
+    }
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        rm(f);
+      }
+    }
+    file.delete();
   }
 
   @Test
@@ -100,9 +114,9 @@ public class BlurOutputFormatTest {
     tableDescriptor.setShardCount(1);
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
-    
-    createShardDirectories(outDir,1);
-    
+
+    createShardDirectories(outDir, 1);
+
     BlurOutputFormat.setupJob(job, tableDescriptor);
 
     assertTrue(job.waitForCompletion(true));
@@ -152,8 +166,8 @@ public class BlurOutputFormatTest {
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
 
-    createShardDirectories(outDir,1);
-    
+    createShardDirectories(outDir, 1);
+
     BlurOutputFormat.setupJob(job, tableDescriptor);
     BlurOutputFormat.setIndexLocally(job, true);
     BlurOutputFormat.setOptimizeInFlight(job, false);
@@ -188,14 +202,14 @@ public class BlurOutputFormatTest {
     FileInputFormat.addInputPath(job, new Path(TEST_ROOT_DIR + "/in"));
     String tableUri = new Path(TEST_ROOT_DIR + "/out").toString();
     CsvBlurMapper.addColumns(job, "cf1", "col");
-      
+
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setShardCount(2);
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
-    
-    createShardDirectories(outDir,2);
-    
+
+    createShardDirectories(outDir, 2);
+
     BlurOutputFormat.setupJob(job, tableDescriptor);
     BlurOutputFormat.setIndexLocally(job, false);
 
@@ -240,8 +254,8 @@ public class BlurOutputFormatTest {
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
 
-    createShardDirectories(outDir,7);
-    
+    createShardDirectories(outDir, 7);
+
     BlurOutputFormat.setupJob(job, tableDescriptor);
     int multiple = 2;
     BlurOutputFormat.setReducerMultiplier(job, multiple);
@@ -264,9 +278,10 @@ public class BlurOutputFormatTest {
     assertEquals(80000, total);
 
   }
-  
-  @Test (expected = IllegalArgumentException.class)
-  public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException,
ClassNotFoundException {
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testBlurOutputFormatValidateReducerCount() throws IOException, InterruptedException,
+      ClassNotFoundException {
     localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true);
     localFs.delete(new Path(TEST_ROOT_DIR + "/out"), true);
     writeRecordsFile("in/part1", 1, 1, 1, 1, "cf1");
@@ -285,14 +300,14 @@ public class BlurOutputFormatTest {
     tableDescriptor.setShardCount(1);
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.setTableUri(tableUri);
-    
-    createShardDirectories(outDir,1);
-    
+
+    createShardDirectories(outDir, 1);
+
     BlurOutputFormat.setupJob(job, tableDescriptor);
     BlurOutputFormat.setReducerMultiplier(job, 2);
     job.setNumReduceTasks(4);
     job.submit();
-    
+
   }
 
   public static String readFile(String name) throws IOException {
@@ -325,12 +340,13 @@ public class BlurOutputFormatTest {
     return file;
   }
 
-  private void createShardDirectories(Path outDir, int shardCount) throws IOException{
+  private void createShardDirectories(Path outDir, int shardCount) throws IOException {
     localFs.mkdirs(outDir);
-    for(int i=0; i<shardCount; i++){
+    for (int i = 0; i < shardCount; i++) {
       localFs.mkdirs(new Path(outDir, BlurUtil.getShardName(i)));
     }
   }
+
   private String getRecord(int rowId, int recordId, String family) {
     return rowId + "," + recordId + "," + family + ",valuetoindex";
   }


Mime
View raw message