Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6C5621013E for ; Tue, 2 Dec 2014 20:36:51 +0000 (UTC) Received: (qmail 95618 invoked by uid 500); 2 Dec 2014 20:36:51 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 95537 invoked by uid 500); 2 Dec 2014 20:36:51 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 94287 invoked by uid 99); 2 Dec 2014 20:36:50 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Dec 2014 20:36:50 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 28A609BBFA4; Tue, 2 Dec 2014 20:36:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Tue, 02 Dec 2014 20:37:07 -0000 Message-Id: <0e1f5a199c474798a39e3bae443ddb78@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [19/50] [abbrv] accumulo git commit: ACCUMULO-3167 Got BulkFileIT working against mini and standalone clusters ACCUMULO-3167 Got BulkFileIT working against mini and standalone clusters Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3342e32d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3342e32d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3342e32d Branch: refs/heads/metrics2 Commit: 3342e32d2d1d1685df43537acd18200782b58d73 Parents: 5534665 Author: Josh Elser Authored: Sun Nov 23 23:43:42 2014 -0500 Committer: Josh Elser Committed: Mon Nov 24 18:08:52 2014 -0500 ---------------------------------------------------------------------- .../accumulo/cluster/AccumuloCluster.java | 9 +++++- .../standalone/StandaloneAccumuloCluster.java | 30 +++++++++++++++++++- .../impl/MiniAccumuloClusterImpl.java | 1 + .../accumulo/harness/AccumuloClusterIT.java | 27 ------------------ .../accumulo/test/functional/BulkFileIT.java | 23 +++++++++++---- 5 files changed, 55 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java index d647835..e823f15 100644 --- a/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java +++ b/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java @@ -16,12 +16,14 @@ */ package org.apache.accumulo.cluster; -import org.apache.accumulo.cluster.ClusterControl; +import java.io.IOException; + import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.hadoop.fs.FileSystem; /** * Defines a minimum required set of methods to run an Accumulo cluster. @@ -73,4 +75,9 @@ public interface AccumuloCluster { * @throws Exception */ void stop() throws Exception; + + /** + * @return the {@link FileSystem} in use by this cluster + */ + FileSystem getFileSystem() throws IOException; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java index 4abb046..58536ed 100644 --- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java +++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java @@ -27,8 +27,12 @@ import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.master.state.SetGoalState; import org.apache.accumulo.minicluster.ServerType; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +44,7 @@ public class StandaloneAccumuloCluster implements AccumuloCluster { private static final Logger log = LoggerFactory.getLogger(StandaloneAccumuloCluster.class); private Instance instance; - private String accumuloHome, accumuloConfDir; + private String accumuloHome, accumuloConfDir, hadoopConfDir; public StandaloneAccumuloCluster(String instanceName, String zookeepers) { this(new ZooKeeperInstance(instanceName, zookeepers)); @@ -66,6 +70,14 @@ public class StandaloneAccumuloCluster implements AccumuloCluster { this.accumuloConfDir = accumuloConfDir; } + public String getHadoopConfDir() { + return hadoopConfDir; + } + + public void setHadoopConfDir(String hadoopConfDir) { + this.hadoopConfDir = hadoopConfDir; + } + @Override public String getInstanceName() { return instance.getInstanceName(); @@ -116,4 +128,20 @@ public class StandaloneAccumuloCluster implements AccumuloCluster { } } + @Override + public FileSystem getFileSystem() throws IOException { + String confDir = hadoopConfDir; + if (null == confDir) { + confDir = System.getenv("HADOOP_CONF_DIR"); + } + if (null == confDir) { + throw new IllegalArgumentException("Cannot determine HADOOP_CONF_DIR for standalone cluster"); + } + // Using CachedConfiguration will make repeatedly calling this method much faster + final Configuration conf = CachedConfiguration.getInstance(); + conf.addResource(new Path(confDir, "core-site.xml")); + // Need hdfs-site.xml for NN HA + conf.addResource(new Path(confDir, "hdfs-site.xml")); + return FileSystem.get(conf); + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java index 54a6468..50e90b8 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java @@ -714,6 +714,7 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { this.getZooKeepers()); } + @Override public FileSystem getFileSystem() { try { return FileSystem.get(new URI(dfsUri), new Configuration()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java index 76bccf5..08c8275 100644 --- a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java +++ b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java @@ -18,22 +18,16 @@ package org.apache.accumulo.harness; import static org.junit.Assert.fail; -import java.util.Map; - import org.apache.accumulo.cluster.AccumuloCluster; import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.admin.SecurityOperations; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; -import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.harness.conf.AccumuloClusterPropertyConfiguration; import org.apache.accumulo.harness.conf.AccumuloStandaloneClusterConfiguration; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -167,27 +161,6 @@ public abstract class AccumuloClusterIT extends AccumuloIT implements MiniCluste } } - @SuppressWarnings("deprecation") - public FileSystem getFileSystem() { - try { - Connector conn = getConnector(); - Map conf = conn.instanceOperations().getSystemConfiguration(); - String uri; - if (conf.containsKey(Property.INSTANCE_VOLUMES.getKey())) { - String instanceVolumes = conf.get(Property.INSTANCE_VOLUMES.getKey()); - uri = StringUtils.split(instanceVolumes, ',')[0]; - } else if (conf.containsKey(Property.INSTANCE_DFS_URI.getKey())) { - uri = conf.get(Property.INSTANCE_DFS_URI.getKey()); - } else { - throw new RuntimeException("No DFS configuraiton present in system configuration"); - } - - return FileSystem.get(new Path(uri).toUri(), new Configuration()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - // TODO Really don't want this here. Will ultimately need to abstract configuration method away from MAConfig // and change over to something more generic @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java index a7cf6bd..3772937 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java @@ -18,6 +18,7 @@ package org.apache.accumulo.test.functional; import static com.google.common.base.Charsets.UTF_8; +import java.io.File; import java.util.Iterator; import java.util.Map.Entry; import java.util.SortedSet; @@ -32,22 +33,22 @@ import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVWriter; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.volume.VolumeConfiguration; +import org.apache.accumulo.harness.AccumuloClusterIT; import org.apache.accumulo.minicluster.MemoryUnit; import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.server.trace.TraceFileSystem; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.junit.Assert; import org.junit.Test; -public class BulkFileIT extends ConfigurableMacIT { +public class BulkFileIT extends AccumuloClusterIT { @Override - public void configure(MiniAccumuloConfigImpl cfg, Configuration conf) { + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration conf) { cfg.setMemory(ServerType.TABLET_SERVER, 128 * 4, MemoryUnit.MEGABYTE); } @@ -67,9 +68,19 @@ public class BulkFileIT extends ConfigurableMacIT { c.tableOperations().addSplits(tableName, splits); Configuration conf = new Configuration(); AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration(); - FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem()); + FileSystem fs = getCluster().getFileSystem(); + + String rootPath; + if (ClusterType.MINI == getClusterType()) { + rootPath = new File(System.getProperty("user.dir"), "target").getCanonicalPath(); + } else if (ClusterType.STANDALONE == getClusterType()) { + rootPath = "/tmp"; + } else { + Assert.fail("Cannot compute root path for test"); + return; + } - String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0]; + String dir = rootPath + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0]; fs.delete(new Path(dir), true);