Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1360010146 for ; Wed, 22 Jan 2014 22:10:33 +0000 (UTC) Received: (qmail 62558 invoked by uid 500); 22 Jan 2014 22:10:16 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 62251 invoked by uid 500); 22 Jan 2014 22:10:10 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 62173 invoked by uid 99); 22 Jan 2014 22:10:09 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jan 2014 22:10:09 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 3590F8A0D07; Wed, 22 Jan 2014 22:10:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Wed, 22 Jan 2014 22:10:11 -0000 Message-Id: <0f21dd7c74ac43d0be71337342928bc7@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [03/10] git commit: ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete Instance. ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete Instance. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/57f9b6cf Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/57f9b6cf Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/57f9b6cf Branch: refs/heads/1.6.0-SNAPSHOT Commit: 57f9b6cfd30b1b2505efbacd2a5ce391dbcd1e0c Parents: 06f8030 Author: Josh Elser Authored: Wed Jan 22 16:27:02 2014 -0500 Committer: Josh Elser Committed: Wed Jan 22 16:27:02 2014 -0500 ---------------------------------------------------------------------- .../core/client/mapreduce/InputFormatBase.java | 11 ++++++++- .../core/client/mapreduce/RangeInputSplit.java | 11 ++++++++- .../test/continuous/ContinuousVerify.java | 24 ++++++++++++++++++-- test/system/continuous/run-verify.sh | 24 +++++++++++++++++++- 4 files changed, 65 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java ---------------------------------------------------------------------- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java index de97f12..9c23246 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java @@ -58,6 +58,8 @@ import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.impl.TabletLocator; import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.client.mock.MockTabletLocator; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.PartialKey; @@ -735,7 +737,14 @@ public abstract class InputFormatBase extends InputFormat { protected static Instance getInstance(Configuration conf) { if (conf.getBoolean(MOCK, false)) return new MockInstance(conf.get(INSTANCE_NAME)); - return new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS)); + + ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS)); + + // Wrap the DefaultConfiguration with a SiteConfiguration + AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration()); + zki.setConfiguration(xmlConfig); + + return zki; } /** http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java ---------------------------------------------------------------------- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java index 01921c4..e1107cc 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java @@ -29,6 +29,8 @@ import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator; import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption; import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; @@ -354,7 +356,14 @@ public class RangeInputSplit extends InputSplit implements Writable { return null; } - return new ZooKeeperInstance(getInstanceName(), getZooKeepers()); + ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers()); + + // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml + // when it's present + AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration()); + zki.setConfiguration(xmlConfig); + + return zki; } public String getInstanceName() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java index d271d22..6546eea 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java +++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java @@ -17,6 +17,7 @@ package org.apache.accumulo.server.test.continuous; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -34,6 +35,8 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.server.test.continuous.ContinuousWalk.BadChecksumException; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -44,12 +47,14 @@ import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.Logger; /** * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined. */ public class ContinuousVerify extends Configured implements Tool { + private static final Logger log = Logger.getLogger(ContinuousVerify.class); public static final VLongWritable DEF = new VLongWritable(-1); @@ -150,9 +155,9 @@ public class ContinuousVerify extends Configured implements Tool { args = argsList.toArray(new String[0]); - if (args.length != 9) { + if (args.length != 10) { throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName() - + " "); + + "
"); } String instance = args[0]; @@ -164,6 +169,7 @@ public class ContinuousVerify extends Configured implements Tool { String maxMaps = args[6]; String reducers = args[7]; boolean scanOffline = Boolean.parseBoolean(args[8]); + String siteFile = args[9]; Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); @@ -210,6 +216,20 @@ public class ContinuousVerify extends Configured implements Tool { job.setOutputFormatClass(TextOutputFormat.class); job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline); + + Path sitePath = new Path(siteFile); + Path siteParentPath = sitePath.getParent(); + if (null == siteParentPath) { + siteParentPath = new Path("/"); + } + + URI siteUri = new URI("hdfs://" + siteFile); + + log.info("Adding " + siteUri + " to DistributedCache"); + + // Make sure that accumulo-site.xml is available for mappers running offline scans + // as they need to correctly choose instance.dfs.dir for the installation + DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration())); TextOutputFormat.setOutputPath(job, new Path(outputdir)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/test/system/continuous/run-verify.sh ---------------------------------------------------------------------- diff --git a/test/system/continuous/run-verify.sh b/test/system/continuous/run-verify.sh index edf58b7..f2b7a25 100755 --- a/test/system/continuous/run-verify.sh +++ b/test/system/continuous/run-verify.sh @@ -24,5 +24,27 @@ if [ -n "$VERIFY_AUTHS" ] ; then AUTH_OPT="--auths $VERIFY_AUTHS"; fi -$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE +if [ ! -r $ACCUMULO_CONF_DIR/accumulo-site.xml ]; then + echo "Could not find accumulo-site.xml in $ACCUMULO_CONF_DIR" + exit 1 +fi + +TARGET_DIR="ci-conf-`date '+%s'`" +hadoop fs -mkdir $TARGET_DIR + +if [ $? -ne 0 ]; then + echo "Could not create $TAGET_DIR in HDFS" + exit 1 +fi + +hadoop fs -put $ACCUMULO_CONF_DIR/accumulo-site.xml ${TARGET_DIR}/ + +if [ $? -ne 0 ]; then + echo "Could not upload accumulo-site.xml to HDFS" + exit 1 +fi + +ABS_DIR="/user/`whoami`/${TARGET_DIR}/accumulo-site.xml" + +$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS --sitefile $ABS_DIR $SCAN_OFFLINE