incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cr...@apache.org
Subject [40/45] git commit: fix multiplier on csv import job
Date Sun, 26 Oct 2014 17:55:38 GMT
fix multiplier on csv import job


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/6b7f2b04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/6b7f2b04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/6b7f2b04

Branch: refs/heads/blur-384-random-port-cleanup
Commit: 6b7f2b049884559c4ee4712bd96403ebee2356d3
Parents: 2683f4b
Author: twilliams <twilliams@apache.org>
Authored: Tue Oct 21 17:11:19 2014 -0400
Committer: twilliams <twilliams@apache.org>
Committed: Tue Oct 21 17:11:19 2014 -0400

----------------------------------------------------------------------
 .../apache/blur/mapreduce/lib/CsvBlurDriver.java  | 10 ++++++----
 .../blur/mapreduce/lib/CsvBlurDriverTest.java     | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6b7f2b04/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
index 6830e32..1814945 100644
--- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
+++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java
@@ -214,10 +214,6 @@ public class CsvBlurDriver {
       int maxDocumentBufferSize = Integer.parseInt(cmd.getOptionValue("b"));
       BlurOutputFormat.setMaxDocumentBufferSize(job, maxDocumentBufferSize);
     }
-    if (cmd.hasOption("r")) {
-      int reducerMultiplier = Integer.parseInt(cmd.getOptionValue("r"));
-      BlurOutputFormat.setReducerMultiplier(job, reducerMultiplier);
-    }
     // processing the 'd' option
     Option[] options = cmd.getOptions();
     for (Option option : options) {
@@ -231,6 +227,12 @@ public class CsvBlurDriver {
       }
     }
     BlurOutputFormat.setupJob(job, tableDescriptor);
+    
+    if (cmd.hasOption("r")) {
+      int reducerMultiplier = Integer.parseInt(cmd.getOptionValue("r"));
+      BlurOutputFormat.setReducerMultiplier(job, reducerMultiplier);
+    }
+    
     BlurMapReduceUtil.addDependencyJars(job.getConfiguration(), Splitter.class);
     return job;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6b7f2b04/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
index 6a673eb..fd6f6c9 100644
--- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
+++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java
@@ -133,6 +133,24 @@ public class CsvBlurDriverTest {
     assertEquals(SnappyCodec.class.getName(), configuration.get(CsvBlurDriver.MAPRED_MAP_OUTPUT_COMPRESSION_CODEC));
   }
 
+  @Test
+  public void multiplierParamShouldIncreaseReduceTasks() throws Exception {
+    Configuration configurationSetup = new Configuration();
+    ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
+      @Override
+      public Iface getClient(String controllerConnectionStr) {
+        return getMockIface();
+      }
+    };
+    int multiplierParam = 10;
+    Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, "-c", "host:40010",
"-d", "family1", "col1",
+        "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1",
"-i", "file:///tmp/test2",
+        "-S", "-C", "1000000", "2000000", "-p", "SNAPPY", "-r", Integer.toString(multiplierParam));
+    assertNotNull(job);
+
+    assertEquals(multiplierParam * shardCount, job.getNumReduceTasks());
+  }
+
   protected Iface getMockIface() {
     InvocationHandler handler = new InvocationHandler() {
 


Mime
View raw message