accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [04/21] git commit: Revert "ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete Instance."
Date Fri, 24 Jan 2014 02:15:37 GMT
Revert "ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete
Instance."

This reverts commit 57f9b6cfd30b1b2505efbacd2a5ce391dbcd1e0c.

Conflicts:
	src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
	src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/66516a0f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/66516a0f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/66516a0f

Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 66516a0f8900f228294cb8d818e350887937b079
Parents: 9cf94f9
Author: Josh Elser <elserj@apache.org>
Authored: Thu Jan 23 17:51:05 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Thu Jan 23 17:51:05 2014 -0500

----------------------------------------------------------------------
 .../core/client/mapreduce/InputFormatBase.java  | 11 +--------
 .../core/client/mapreduce/RangeInputSplit.java  | 11 +--------
 .../test/continuous/ContinuousVerify.java       | 24 ++------------------
 test/system/continuous/run-verify.sh            | 24 +-------------------
 4 files changed, 5 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 705ddbd..de97f12 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -58,8 +58,6 @@ import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.client.impl.TabletLocator;
 import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.client.mock.MockTabletLocator;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.PartialKey;
@@ -737,14 +735,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
   protected static Instance getInstance(Configuration conf) {
     if (conf.getBoolean(MOCK, false))
       return new MockInstance(conf.get(INSTANCE_NAME));
-
-    ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
-
-    // Wrap the DefaultConfiguration with a SiteConfiguration
-    AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
-    zki.setConfiguration(xmlConfig);
-
-    return zki;
+    return new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index e1107cc..01921c4 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@ -29,8 +29,6 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
 import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
 import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.PartialKey;
@@ -356,14 +354,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
       return null;
     }
     
-    ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
-
-    // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
-    // when it's present
-    AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
-    zki.setConfiguration(xmlConfig);
-
-    return zki;
+    return new ZooKeeperInstance(getInstanceName(), getZooKeepers());
   }
 
   public String getInstanceName() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index 4a5c0cf..6165d2a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -17,7 +17,6 @@
 package org.apache.accumulo.server.test.continuous;
 
 import java.io.IOException;
-import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -35,8 +34,6 @@ import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.server.test.continuous.ContinuousWalk.BadChecksumException;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -47,14 +44,12 @@ import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Logger;
 
 /**
  * A map reduce job that verifies a table created by continuous ingest. It verifies that
all referenced nodes are defined.
  */
 
 public class ContinuousVerify extends Configured implements Tool {
-  private static final Logger log = Logger.getLogger(ContinuousVerify.class);
 
   public static final VLongWritable DEF = new VLongWritable(-1);
 
@@ -155,9 +150,9 @@ public class ContinuousVerify extends Configured implements Tool {
 
     args = argsList.toArray(new String[0]);
 
-    if (args.length != 10) {
+    if (args.length != 9) {
       throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
-          + " <instance name> <zookeepers> <user> <pass> <table>
<output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
+          + " <instance name> <zookeepers> <user> <pass> <table>
<output dir> <max mappers> <num reducers> <scan offline>");
     }
 
     String instance = args[0];
@@ -169,7 +164,6 @@ public class ContinuousVerify extends Configured implements Tool {
     String maxMaps = args[6];
     String reducers = args[7];
     boolean scanOffline = Boolean.parseBoolean(args[8]);
-    String siteFile = args[9];
 
     Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
     job.setJarByClass(this.getClass());
@@ -217,20 +211,6 @@ public class ContinuousVerify extends Configured implements Tool {
 
     job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
 
-    Path sitePath = new Path(siteFile);
-    Path siteParentPath = sitePath.getParent();
-    if (null == siteParentPath) {
-      siteParentPath = new Path("/");
-    }
-
-    URI siteUri = new URI("hdfs://" + siteFile);
-
-    log.info("Adding " + siteUri + " to DistributedCache");
-
-    // Make sure that accumulo-site.xml is available for mappers running offline scans
-    // as they need to correctly choose instance.dfs.dir for the installation
-    DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri,
job.getConfiguration()));
-
     TextOutputFormat.setOutputPath(job, new Path(outputdir));
 
     job.waitForCompletion(true);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/test/system/continuous/run-verify.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/run-verify.sh b/test/system/continuous/run-verify.sh
index f2b7a25..edf58b7 100755
--- a/test/system/continuous/run-verify.sh
+++ b/test/system/continuous/run-verify.sh
@@ -24,27 +24,5 @@ if [ -n "$VERIFY_AUTHS" ] ; then
 	AUTH_OPT="--auths $VERIFY_AUTHS";
 fi
 
-if [ ! -r $ACCUMULO_CONF_DIR/accumulo-site.xml ]; then
-    echo "Could not find accumulo-site.xml in $ACCUMULO_CONF_DIR"
-    exit 1
-fi
-
-TARGET_DIR="ci-conf-`date '+%s'`"
-hadoop fs -mkdir $TARGET_DIR
-
-if [ $? -ne 0 ]; then
-    echo "Could not create $TAGET_DIR in HDFS"
-    exit 1
-fi
-
-hadoop fs -put $ACCUMULO_CONF_DIR/accumulo-site.xml ${TARGET_DIR}/
-
-if [ $? -ne 0 ]; then
-    echo "Could not upload accumulo-site.xml to HDFS"
-    exit 1
-fi
-
-ABS_DIR="/user/`whoami`/${TARGET_DIR}/accumulo-site.xml"
-
-$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify
-libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT
$VERIFY_MAX_MAPS $VERIFY_REDUCERS --sitefile $ABS_DIR  $SCAN_OFFLINE
+$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify
-libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT
$VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE
 


Mime
View raw message