incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Updating the map reduce incremental driver. If not source paths then exit early, adding reducer multipler as option and the ability to add extra config files to running config.
Date Tue, 28 Apr 2015 00:50:24 GMT
Updating the map reduce incremental driver.  If not source paths then exit early, adding reducer
multipler as option and the ability to add extra config files to running config.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8e36828b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8e36828b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8e36828b

Branch: refs/heads/master
Commit: 8e36828b0029c2adfd5139c2b3b2a6e84a8b392c
Parents: beb6c89
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Apr 27 20:49:53 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Apr 27 20:49:53 2015 -0400

----------------------------------------------------------------------
 .../blur/mapreduce/lib/update/Driver.java       | 24 ++++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8e36828b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/Driver.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/Driver.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/Driver.java
index 05ee5a2..fb7d570 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/Driver.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/Driver.java
@@ -57,10 +57,19 @@ public class Driver extends Configured implements Tool {
   @Override
   public int run(String[] args) throws Exception {
     int c = 0;
+    if (args.length < 5) {
+      System.err
+          .println("Usage Driver <table> <mr inc working path> <output path>
<zk connection> <reducer multipler> <extra config files...>");
+    }
     String table = args[c++];
     String mrIncWorkingPathStr = args[c++];
     String outputPathStr = args[c++];
     String blurZkConnection = args[c++];
+    int reducerMultipler = Integer.parseInt(args[c++]);
+    for (; c < args.length; c++) {
+      String externalConfigFileToAdd = args[c];
+      getConf().addResource(new Path(externalConfigFileToAdd));
+    }
 
     Path outputPath = new Path(outputPathStr);
     Path mrIncWorkingPath = new Path(mrIncWorkingPathStr);
@@ -76,14 +85,17 @@ public class Driver extends Configured implements Tool {
     fileSystem.mkdirs(completeData);
     fileSystem.mkdirs(fileCache);
 
+    List<Path> srcPathList = new ArrayList<Path>();
+    for (FileStatus fileStatus : fileSystem.listStatus(newData)) {
+      srcPathList.add(fileStatus.getPath());
+    }
+    if (srcPathList.isEmpty()) {
+      return 0;
+    }
+
     List<Path> inprogressPathList = new ArrayList<Path>();
     boolean success = false;
     try {
-      List<Path> srcPathList = new ArrayList<Path>();
-      for (FileStatus fileStatus : fileSystem.listStatus(newData)) {
-        srcPathList.add(fileStatus.getPath());
-      }
-
       inprogressPathList = movePathList(fileSystem, inprogressData, srcPathList);
 
       Job job = Job.getInstance(getConf(), "Blur Row Updater for table [" + table + "]");
@@ -111,6 +123,8 @@ public class Driver extends Configured implements Tool {
       job.setPartitionerClass(IndexKeyPartitioner.class);
       job.setGroupingComparatorClass(IndexKeyWritableComparator.class);
 
+      BlurOutputFormat.setReducerMultiplier(job, reducerMultipler);
+
       success = job.waitForCompletion(true);
       Counters counters = job.getCounters();
       LOG.info("Counters [" + counters + "]");


Mime
View raw message