lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1558545 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/contrib/ solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/ solr/example/ solr/example/scripts/map-reduce/
Date Wed, 15 Jan 2014 19:37:19 GMT
Author: markrmiller
Date: Wed Jan 15 19:37:19 2014
New Revision: 1558545

URL: http://svn.apache.org/r1558545
Log:
SOLR-1301: Clean up.

Added:
    lucene/dev/branches/branch_4x/solr/example/scripts/map-reduce/set-map-reduce-classpath.sh
      - copied unchanged from r1547962, lucene/dev/trunk/solr/example/scripts/map-reduce/set-map-reduce-classpath.sh
Removed:
    lucene/dev/branches/branch_4x/solr/example/scripts/map-reduce/map-reduce-indexer.bat
    lucene/dev/branches/branch_4x/solr/example/scripts/map-reduce/map-reduce-indexer.sh
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
    lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
    lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)

Modified: lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java?rev=1558545&r1=1558544&r2=1558545&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
(original)
+++ lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
Wed Jan 15 19:37:19 2014
@@ -136,7 +136,7 @@ public class MapReduceIndexerTool extend
       showNonSolrCloud = Arrays.asList(args).contains(SHOW_NON_SOLR_CLOUD); // intercept
it first
       
       ArgumentParser parser = ArgumentParsers
-        .newArgumentParser("hadoop [GenericOptions]... jar search-mr-*-job.jar " + MapReduceIndexerTool.class.getName(),
false)
+        .newArgumentParser("hadoop [GenericOptions]... jar solr-map-reduce-*.jar ", false)
         .defaultHelp(true)
         .description(
           "MapReduce batch job driver that takes a morphline and creates a set of Solr index
shards from a set of input files " +
@@ -197,7 +197,7 @@ public class MapReduceIndexerTool extend
               "# (Re)index an Avro based Twitter tweet file:\n" +
               "sudo -u hdfs hadoop \\\n" + 
               "  --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
-              "  jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName()
+ " \\\n" +
+              "  jar target/solr-map-reduce-*.jar \\\n" +
               "  -D 'mapred.child.java.opts=-Xmx500m' \\\n" + 
 //            "  -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + 
               "  --log4j src/test/resources/log4j.properties \\\n" + 
@@ -213,7 +213,7 @@ public class MapReduceIndexerTool extend
               "# 3) file was last modified less than 100000 minutes ago\n" +
               "# 4) file size is between 1 MB and 1 GB\n" +
               "# Also include extra library jar file containing JSON tweet Java parser:\n"
+
-              "hadoop jar target/search-mr-*-job.jar " + "com.cloudera.cdk.morphline.hadoop.find.HdfsFindTool"
+ " \\\n" + 
+              "hadoop jar target/solr-map-reduce-*.jar " + "com.cloudera.cdk.morphline.hadoop.find.HdfsFindTool"
+ " \\\n" + 
               "  -find hdfs:///user/$USER/solrloadtest/twitter/tweets \\\n" + 
               "  -type f \\\n" + 
               "  -name 'sample-statuses*.gz' \\\n" + 
@@ -222,7 +222,7 @@ public class MapReduceIndexerTool extend
               "  -size +1000000c \\\n" + 
               "| sudo -u hdfs hadoop \\\n" + 
               "  --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" + 
-              "  jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName()
+ " \\\n" +
+              "  jar target/solr-map-reduce-*.jar \\\n" +
               "  -D 'mapred.child.java.opts=-Xmx500m' \\\n" + 
 //            "  -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + 
               "  --log4j src/test/resources/log4j.properties \\\n" + 
@@ -236,7 +236,7 @@ public class MapReduceIndexerTool extend
               "# (explicitly specify Solr URLs - for a SolrCloud cluster see next example):\n"
+
               "sudo -u hdfs hadoop \\\n" + 
               "  --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
-              "  jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName()
+ " \\\n" +
+              "  jar target/solr-map-reduce-*.jar \\\n" +
               "  -D 'mapred.child.java.opts=-Xmx500m' \\\n" + 
 //            "  -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + 
               "  --log4j src/test/resources/log4j.properties \\\n" + 
@@ -252,7 +252,7 @@ public class MapReduceIndexerTool extend
               "# (discover shards and Solr URLs through ZooKeeper):\n" +
               "sudo -u hdfs hadoop \\\n" + 
               "  --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
-              "  jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName()
+ " \\\n" +
+              "  jar target/solr-map-reduce-*.jar \\\n" +
               "  -D 'mapred.child.java.opts=-Xmx500m' \\\n" + 
 //            "  -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + 
               "  --log4j src/test/resources/log4j.properties \\\n" + 

Modified: lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java?rev=1558545&r1=1558544&r2=1558545&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
(original)
+++ lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
Wed Jan 15 19:37:19 2014
@@ -38,17 +38,14 @@ import org.apache.hadoop.util.JarFinder;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.cloud.AbstractZkTestCase;
 import org.apache.solr.hadoop.hack.MiniMRCluster;
-import org.apache.solr.handler.extraction.ExtractingParams;
 import org.apache.solr.util.ExternalPaths;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
@@ -324,7 +321,6 @@ public class MorphlineBasicMiniMRTest ex
     jobConf.setMaxMapAttempts(1);
     jobConf.setMaxReduceAttempts(1);
     jobConf.setJar(SEARCH_ARCHIVES_JAR);
-    jobConf.setBoolean(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);
     
     int shards = 2;
     int maxReducers = Integer.MAX_VALUE;

Modified: lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java?rev=1558545&r1=1558544&r2=1558545&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
(original)
+++ lucene/dev/branches/branch_4x/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
Wed Jan 15 19:37:19 2014
@@ -67,7 +67,6 @@ import org.apache.solr.common.params.Mod
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.hadoop.hack.MiniMRClientCluster;
 import org.apache.solr.hadoop.hack.MiniMRClientClusterFactory;
-import org.apache.solr.handler.extraction.ExtractingParams;
 import org.apache.solr.util.ExternalPaths;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -367,7 +366,6 @@ public class MorphlineGoLiveMiniMRTest e
     jobConf.setMaxMapAttempts(1);
     jobConf.setMaxReduceAttempts(1);
     jobConf.setJar(SEARCH_ARCHIVES_JAR);
-    jobConf.setBoolean(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);
 
     MapReduceIndexerTool tool;
     int res;



Mime
View raw message