accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [01/10] git commit: ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete Instance.
Date Wed, 22 Jan 2014 22:10:09 GMT
Updated Branches:
  refs/heads/1.4.5-SNAPSHOT 06f80305e -> 57f9b6cfd
  refs/heads/1.5.1-SNAPSHOT bd67c465f -> 7bef40489
  refs/heads/1.6.0-SNAPSHOT b642d1be7 -> 2e6581769
  refs/heads/master 680433bec -> 996fe113c


ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete
Instance.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/57f9b6cf
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/57f9b6cf
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/57f9b6cf

Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 57f9b6cfd30b1b2505efbacd2a5ce391dbcd1e0c
Parents: 06f8030
Author: Josh Elser <elserj@apache.org>
Authored: Wed Jan 22 16:27:02 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Wed Jan 22 16:27:02 2014 -0500

----------------------------------------------------------------------
 .../core/client/mapreduce/InputFormatBase.java  | 11 ++++++++-
 .../core/client/mapreduce/RangeInputSplit.java  | 11 ++++++++-
 .../test/continuous/ContinuousVerify.java       | 24 ++++++++++++++++++--
 test/system/continuous/run-verify.sh            | 24 +++++++++++++++++++-
 4 files changed, 65 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index de97f12..9c23246 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -58,6 +58,8 @@ import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.client.impl.TabletLocator;
 import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.client.mock.MockTabletLocator;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.PartialKey;
@@ -735,7 +737,14 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
   protected static Instance getInstance(Configuration conf) {
     if (conf.getBoolean(MOCK, false))
       return new MockInstance(conf.get(INSTANCE_NAME));
-    return new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
+    
+    ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
+
+    // Wrap the DefaultConfiguration with a SiteConfiguration
+    AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+    zki.setConfiguration(xmlConfig);
+
+    return zki;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 01921c4..e1107cc 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@ -29,6 +29,8 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
 import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
 import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.PartialKey;
@@ -354,7 +356,14 @@ public class RangeInputSplit extends InputSplit implements Writable {
       return null;
     }
     
-    return new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+    ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+
+    // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
+    // when it's present
+    AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+    zki.setConfiguration(xmlConfig);
+
+    return zki;
   }
 
   public String getInstanceName() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index d271d22..6546eea 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -17,6 +17,7 @@
 package org.apache.accumulo.server.test.continuous;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -34,6 +35,8 @@ import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.server.test.continuous.ContinuousWalk.BadChecksumException;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -44,12 +47,14 @@ import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
 
 /**
  * A map reduce job that verifies a table created by continuous ingest. It verifies that
all referenced nodes are defined.
  */
 
 public class ContinuousVerify extends Configured implements Tool {
+  private static final Logger log = Logger.getLogger(ContinuousVerify.class);
   
   public static final VLongWritable DEF = new VLongWritable(-1);
   
@@ -150,9 +155,9 @@ public class ContinuousVerify extends Configured implements Tool {
     
     args = argsList.toArray(new String[0]);
 
-    if (args.length != 9) {
+    if (args.length != 10) {
       throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
-          + " <instance name> <zookeepers> <user> <pass> <table>
<output dir> <max mappers> <num reducers> <scan offline>");
+          + " <instance name> <zookeepers> <user> <pass> <table>
<output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
     }
     
     String instance = args[0];
@@ -164,6 +169,7 @@ public class ContinuousVerify extends Configured implements Tool {
     String maxMaps = args[6];
     String reducers = args[7];
     boolean scanOffline = Boolean.parseBoolean(args[8]);
+    String siteFile = args[9];
     
     Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
     job.setJarByClass(this.getClass());
@@ -210,6 +216,20 @@ public class ContinuousVerify extends Configured implements Tool {
     job.setOutputFormatClass(TextOutputFormat.class);
     
     job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
+    
+    Path sitePath = new Path(siteFile);
+    Path siteParentPath = sitePath.getParent();
+    if (null == siteParentPath) {
+      siteParentPath = new Path("/");
+    }
+    
+    URI siteUri = new URI("hdfs://" + siteFile);
+    
+    log.info("Adding " + siteUri + " to DistributedCache");
+    
+    // Make sure that accumulo-site.xml is available for mappers running offline scans
+    // as they need to correctly choose instance.dfs.dir for the installation
+    DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri,
job.getConfiguration()));
 
     TextOutputFormat.setOutputPath(job, new Path(outputdir));
     

http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/test/system/continuous/run-verify.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/run-verify.sh b/test/system/continuous/run-verify.sh
index edf58b7..f2b7a25 100755
--- a/test/system/continuous/run-verify.sh
+++ b/test/system/continuous/run-verify.sh
@@ -24,5 +24,27 @@ if [ -n "$VERIFY_AUTHS" ] ; then
 	AUTH_OPT="--auths $VERIFY_AUTHS";
 fi
 
-$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify
-libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT
$VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE
+if [ ! -r $ACCUMULO_CONF_DIR/accumulo-site.xml ]; then
+    echo "Could not find accumulo-site.xml in $ACCUMULO_CONF_DIR"
+    exit 1
+fi
+
+TARGET_DIR="ci-conf-`date '+%s'`"
+hadoop fs -mkdir $TARGET_DIR
+
+if [ $? -ne 0 ]; then
+    echo "Could not create $TAGET_DIR in HDFS"
+    exit 1
+fi
+
+hadoop fs -put $ACCUMULO_CONF_DIR/accumulo-site.xml ${TARGET_DIR}/
+
+if [ $? -ne 0 ]; then
+    echo "Could not upload accumulo-site.xml to HDFS"
+    exit 1
+fi
+
+ABS_DIR="/user/`whoami`/${TARGET_DIR}/accumulo-site.xml"
+
+$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify
-libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT
$VERIFY_MAX_MAPS $VERIFY_REDUCERS --sitefile $ABS_DIR  $SCAN_OFFLINE
 


Mime
View raw message