incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject svn commit: r1245758 - in /incubator/accumulo/trunk: ./ conf/ src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/
Date Fri, 17 Feb 2012 20:56:12 GMT
Author: vines
Date: Fri Feb 17 20:56:11 2012
New Revision: 1245758

URL: http://svn.apache.org/viewvc?rev=1245758&view=rev
Log:
merging ACCUMULO-390


Added:
    incubator/accumulo/trunk/conf/accumulo-env.sh.1GBstandalone-default-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.1GBstandalone-default-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.1GBstandalone-native-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.1GBstandalone-native-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.2GBstandalone-default-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.2GBstandalone-default-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.2GBstandalone-native-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.2GBstandalone-native-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.3GBcluster-default-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.3GBcluster-default-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.3GBcluster-native-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.3GBcluster-native-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.512MBBstandalone-default-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.512MBBstandalone-default-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.512MBBstandalone-native-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-env.sh.512MBBstandalone-native-example
    incubator/accumulo/trunk/conf/accumulo-site.xml.1GBstandalone-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-site.xml.1GBstandalone-example
    incubator/accumulo/trunk/conf/accumulo-site.xml.2GBstandalone-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-site.xml.2GBstandalone-example
    incubator/accumulo/trunk/conf/accumulo-site.xml.3GBcluster-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-site.xml.3GBcluster-example
    incubator/accumulo/trunk/conf/accumulo-site.xml.512MBBstandalone-example
      - copied unchanged from r1245757, incubator/accumulo/branches/1.4/conf/accumulo-site.xml.512MBBstandalone-example
Removed:
    incubator/accumulo/trunk/conf/accumulo-env.sh.1GBstandalone-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.2GBstandalone-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.3GBcluster-example
    incubator/accumulo/trunk/conf/accumulo-env.sh.512MBBstandalone-example
    incubator/accumulo/trunk/conf/accumulo-site.xml.example
Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
    incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 17 20:56:11 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632
 /incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1245631,1245655
+/incubator/accumulo/branches/1.4:1201902-1245757

Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java?rev=1245758&r1=1245757&r2=1245758&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
(original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
Fri Feb 17 20:56:11 2012
@@ -56,6 +56,7 @@ public class WikipediaConfiguration {
   public final static String BULK_INGEST_DIR = "wikipedia.bulk.ingest.dir";
   public final static String BULK_INGEST_FAILURE_DIR = "wikipedia.bulk.ingest.failure.dir";
   public final static String BULK_INGEST_BUFFER_SIZE = "wikipedia.bulk.ingest.buffer.size";
+  public final static String PARTITIONED_INPUT_MIN_SPLIT_SIZE = "wikipedia.min.input.split.size";
   
   
   public static String getUser(Configuration conf) {
@@ -130,6 +131,10 @@ public class WikipediaConfiguration {
     return new Path(conf.get(PARTITIONED_ARTICLES_DIRECTORY));
   }
   
+  public static long getMinInputSplitSize(Configuration conf) {
+    return conf.getLong(PARTITIONED_INPUT_MIN_SPLIT_SIZE, 1l << 27);
+  }
+
   public static boolean runPartitioner(Configuration conf) {
     return conf.getBoolean(RUN_PARTITIONER, false);
   }

Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java?rev=1245758&r1=1245757&r2=1245758&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
(original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
Fri Feb 17 20:56:11 2012
@@ -217,7 +217,8 @@ public class WikipediaPartitionedIngeste
     // setup input format
     ingestJob.setInputFormatClass(SequenceFileInputFormat.class);
     SequenceFileInputFormat.setInputPaths(ingestJob, WikipediaConfiguration.getPartitionedArticlesPath(ingestConf));
-    SequenceFileInputFormat.setMinInputSplitSize(ingestJob, 1l << 28);
+    // TODO make split size configurable
+    SequenceFileInputFormat.setMinInputSplitSize(ingestJob, WikipediaConfiguration.getMinInputSplitSize(ingestConf));
 
     // setup output format
     ingestJob.setMapOutputKeyClass(Text.class);



Mime
View raw message