accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [27/50] [abbrv] accumulo git commit: ACCUMULO-3167 Got BulkFileIT working against mini and standalone clusters
Date Wed, 26 Nov 2014 04:23:43 GMT
ACCUMULO-3167 Got BulkFileIT working against mini and standalone clusters


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3342e32d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3342e32d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3342e32d

Branch: refs/heads/master
Commit: 3342e32d2d1d1685df43537acd18200782b58d73
Parents: 5534665
Author: Josh Elser <elserj@apache.org>
Authored: Sun Nov 23 23:43:42 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Mon Nov 24 18:08:52 2014 -0500

----------------------------------------------------------------------
 .../accumulo/cluster/AccumuloCluster.java       |  9 +++++-
 .../standalone/StandaloneAccumuloCluster.java   | 30 +++++++++++++++++++-
 .../impl/MiniAccumuloClusterImpl.java           |  1 +
 .../accumulo/harness/AccumuloClusterIT.java     | 27 ------------------
 .../accumulo/test/functional/BulkFileIT.java    | 23 +++++++++++----
 5 files changed, 55 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
index d647835..e823f15 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
@@ -16,12 +16,14 @@
  */
 package org.apache.accumulo.cluster;
 
-import org.apache.accumulo.cluster.ClusterControl;
+import java.io.IOException;
+
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientConfiguration;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.hadoop.fs.FileSystem;
 
 /**
  * Defines a minimum required set of methods to run an Accumulo cluster.
@@ -73,4 +75,9 @@ public interface AccumuloCluster {
    * @throws Exception
    */
   void stop() throws Exception;
+
+  /**
+   * @return the {@link FileSystem} in use by this cluster
+   */
+  FileSystem getFileSystem() throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
index 4abb046..58536ed 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
@@ -27,8 +27,12 @@ import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.master.state.SetGoalState;
 import org.apache.accumulo.minicluster.ServerType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,7 +44,7 @@ public class StandaloneAccumuloCluster implements AccumuloCluster {
   private static final Logger log = LoggerFactory.getLogger(StandaloneAccumuloCluster.class);
 
   private Instance instance;
-  private String accumuloHome, accumuloConfDir;
+  private String accumuloHome, accumuloConfDir, hadoopConfDir;
 
   public StandaloneAccumuloCluster(String instanceName, String zookeepers) {
     this(new ZooKeeperInstance(instanceName, zookeepers));
@@ -66,6 +70,14 @@ public class StandaloneAccumuloCluster implements AccumuloCluster {
     this.accumuloConfDir = accumuloConfDir;
   }
 
+  public String getHadoopConfDir() {
+    return hadoopConfDir;
+  }
+
+  public void setHadoopConfDir(String hadoopConfDir) {
+    this.hadoopConfDir = hadoopConfDir;
+  }
+
   @Override
   public String getInstanceName() {
     return instance.getInstanceName();
@@ -116,4 +128,20 @@ public class StandaloneAccumuloCluster implements AccumuloCluster {
     }
   }
 
+  @Override
+  public FileSystem getFileSystem() throws IOException {
+    String confDir = hadoopConfDir;
+    if (null == confDir) {
+      confDir = System.getenv("HADOOP_CONF_DIR");
+    }
+    if (null == confDir) {
+      throw new IllegalArgumentException("Cannot determine HADOOP_CONF_DIR for standalone
cluster");
+    }
+    // Using CachedConfiguration will make repeatedly calling this method much faster
+    final Configuration conf = CachedConfiguration.getInstance();
+    conf.addResource(new Path(confDir, "core-site.xml"));
+    // Need hdfs-site.xml for NN HA
+    conf.addResource(new Path(confDir, "hdfs-site.xml"));
+    return FileSystem.get(conf);
+  }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index 54a6468..50e90b8 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@ -714,6 +714,7 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster {
         this.getZooKeepers());
   }
 
+  @Override
   public FileSystem getFileSystem() {
     try {
       return FileSystem.get(new URI(dfsUri), new Configuration());

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
index 76bccf5..08c8275 100644
--- a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
+++ b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
@@ -18,22 +18,16 @@ package org.apache.accumulo.harness;
 
 import static org.junit.Assert.fail;
 
-import java.util.Map;
-
 import org.apache.accumulo.cluster.AccumuloCluster;
 import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.admin.SecurityOperations;
 import org.apache.accumulo.core.client.admin.TableOperations;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.harness.conf.AccumuloClusterPropertyConfiguration;
 import org.apache.accumulo.harness.conf.AccumuloStandaloneClusterConfiguration;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.junit.After;
 import org.junit.Assume;
 import org.junit.Before;
@@ -167,27 +161,6 @@ public abstract class AccumuloClusterIT extends AccumuloIT implements
MiniCluste
     }
   }
 
-  @SuppressWarnings("deprecation")
-  public FileSystem getFileSystem() {
-    try {
-      Connector conn = getConnector();
-      Map<String,String> conf = conn.instanceOperations().getSystemConfiguration();
-      String uri;
-      if (conf.containsKey(Property.INSTANCE_VOLUMES.getKey())) {
-        String instanceVolumes = conf.get(Property.INSTANCE_VOLUMES.getKey());
-        uri = StringUtils.split(instanceVolumes, ',')[0];
-      } else if (conf.containsKey(Property.INSTANCE_DFS_URI.getKey())) {
-        uri = conf.get(Property.INSTANCE_DFS_URI.getKey());
-      } else {
-        throw new RuntimeException("No DFS configuraiton present in system configuration");
-      }
-
-      return FileSystem.get(new Path(uri).toUri(), new Configuration());
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   // TODO Really don't want this here. Will ultimately need to abstract configuration method
away from MAConfig
   // and change over to something more generic
   @Override

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index a7cf6bd..3772937 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.test.functional;
 
 import static com.google.common.base.Charsets.UTF_8;
 
+import java.io.File;
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.SortedSet;
@@ -32,22 +33,22 @@ import org.apache.accumulo.core.file.FileOperations;
 import org.apache.accumulo.core.file.FileSKVWriter;
 import org.apache.accumulo.core.file.rfile.RFile;
 import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.harness.AccumuloClusterIT;
 import org.apache.accumulo.minicluster.MemoryUnit;
 import org.apache.accumulo.minicluster.ServerType;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.trace.TraceFileSystem;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.junit.Assert;
 import org.junit.Test;
 
-public class BulkFileIT extends ConfigurableMacIT {
+public class BulkFileIT extends AccumuloClusterIT {
 
   @Override
-  public void configure(MiniAccumuloConfigImpl cfg, Configuration conf) {
+  public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration conf) {
     cfg.setMemory(ServerType.TABLET_SERVER, 128 * 4, MemoryUnit.MEGABYTE);
   }
 
@@ -67,9 +68,19 @@ public class BulkFileIT extends ConfigurableMacIT {
     c.tableOperations().addSplits(tableName, splits);
     Configuration conf = new Configuration();
     AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
-    FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
+    FileSystem fs = getCluster().getFileSystem();
+
+    String rootPath;
+    if (ClusterType.MINI == getClusterType()) {
+      rootPath = new File(System.getProperty("user.dir"), "target").getCanonicalPath();
+    } else if (ClusterType.STANDALONE == getClusterType()) {
+      rootPath = "/tmp";
+    } else {
+      Assert.fail("Cannot compute root path for test");
+      return;
+    }
 
-    String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
+    String dir = rootPath + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
 
     fs.delete(new Path(dir), true);
 


Mime
View raw message