accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject git commit: ACCUMULO-1409 Make Mini support loading native maps
Date Wed, 20 Nov 2013 22:56:42 GMT
Updated Branches:
  refs/heads/1.6.0-SNAPSHOT 8f5dc6107 -> 211ca036c


ACCUMULO-1409 Make Mini support loading native maps


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/211ca036
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/211ca036
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/211ca036

Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 211ca036c9cbf2c9aec2a13496dc27db758a19db
Parents: 8f5dc61
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Wed Nov 20 17:53:45 2013 -0500
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Wed Nov 20 17:53:45 2013 -0500

----------------------------------------------------------------------
 .../minicluster/MiniAccumuloCluster.java        | 29 ++++++++------------
 .../minicluster/MiniAccumuloConfig.java         | 22 +++++++++++++++
 .../test/functional/ConfigurableMacIT.java      |  3 ++
 .../accumulo/test/functional/NativeMapIT.java   | 10 +++++--
 4 files changed, 43 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 8e7e2ba..0a50747 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -53,6 +53,7 @@ import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.master.thrift.MasterGoalState;
 import org.apache.accumulo.core.util.Daemon;
 import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.gc.SimpleGarbageCollector;
 import org.apache.accumulo.master.Master;
@@ -61,10 +62,9 @@ import org.apache.accumulo.server.init.Initialize;
 import org.apache.accumulo.server.util.PortUtils;
 import org.apache.accumulo.server.util.time.SimpleTimer;
 import org.apache.accumulo.start.Main;
-import org.apache.commons.configuration.MapConfiguration;
 import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
 import org.apache.accumulo.tserver.TabletServer;
-import org.apache.commons.io.FileUtils;
+import org.apache.commons.configuration.MapConfiguration;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
 import org.apache.hadoop.conf.Configuration;
@@ -235,7 +235,6 @@ public class MiniAccumuloCluster {
 
     ArrayList<String> argList = new ArrayList<String>();
     argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath));
-    argList.add("-Djava.library.path=" + config.getLibDir());
     argList.addAll(extraJvmOpts);
     for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) {
       argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue()));
@@ -248,6 +247,9 @@ public class MiniAccumuloCluster {
     builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
     builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
     builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath());
+    String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator);
+    builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath);
+    builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath);
 
     // if we're running under accumulo.start, we forward these env vars
     String env = System.getenv("HADOOP_PREFIX");
@@ -353,6 +355,7 @@ public class MiniAccumuloCluster {
     File clientConfFile = config.getClientConfFile();
     // Write only the properties that correspond to ClientConfiguration properties
     writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new
Predicate<Entry<String,String>>() {
+      @Override
       public boolean apply(Entry<String,String> v) {
         return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null;
       }
@@ -375,18 +378,6 @@ public class MiniAccumuloCluster {
     zooCfg.store(fileWriter, null);
 
     fileWriter.close();
-
-    File nativeMap = new File(config.getLibDir().getAbsolutePath() + "/native/map");
-    nativeMap.mkdirs();
-    File testRoot = new File(new File(new File(System.getProperty("user.dir")).getParent()
+ "/server/src/main/c++/nativeMap").getAbsolutePath());
-
-    if (testRoot.exists()) {
-      for (String file : testRoot.list()) {
-        File src = new File(testRoot, file);
-        if (src.isFile() && file.startsWith("libNativeMap"))
-          FileUtils.copyFile(src, new File(nativeMap, file));
-      }
-    }
   }
 
   private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings)
throws IOException {
@@ -400,6 +391,7 @@ public class MiniAccumuloCluster {
     fileWriter.append("</configuration>\n");
     fileWriter.close();
   }
+
   private void writeConfigProperties(File file, Map<String,String> settings) throws
IOException {
     FileWriter fileWriter = new FileWriter(file);
 
@@ -439,7 +431,7 @@ public class MiniAccumuloCluster {
     if (!initialized) {
       // sleep a little bit to let zookeeper come up before calling init, seems to work better
       long startTime = System.currentTimeMillis();
-      while (true) { 
+      while (true) {
         try {
           Socket s = new Socket("localhost", config.getZooKeeperPort());
           s.getOutputStream().write("ruok\n".getBytes());
@@ -449,7 +441,7 @@ public class MiniAccumuloCluster {
           if (n == 4 && new String(buffer, 0, n).equals("imok"))
             break;
         } catch (Exception e) {
-          if(System.currentTimeMillis() - startTime >= 10000) {
+          if (System.currentTimeMillis() - startTime >= 10000) {
             throw new RuntimeException("Zookeeper did not start within 10 seconds . Check
the logs in " + config.getLogDir() + " for errors.");
           }
           UtilWaitThread.sleep(250);
@@ -626,7 +618,8 @@ public class MiniAccumuloCluster {
   }
 
   public ClientConfiguration getClientConfig() {
-    return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(this.getZooKeepers());
+    return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(
+        this.getZooKeepers());
   }
 
   public FileSystem getFileSystem() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 1af2a1e..2326f95 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -61,6 +61,8 @@ public class MiniAccumuloConfig {
 
   private String[] classpathItems = null;
 
+  private String[] nativePathItems = null;
+
   /**
    * @param dir
    *          An empty or nonexistant directory that Accumulo and Zookeeper can store data
in. Creating the directory is left to the user. Java 7, Guava, and
@@ -425,6 +427,26 @@ public class MiniAccumuloConfig {
   }
 
   /**
+   * @return the paths to use for loading native libraries
+   * 
+   * @since 1.6.0
+   */
+  public String[] getNativeLibPaths() {
+    return this.nativePathItems == null ? new String[0] : this.nativePathItems;
+  }
+
+  /**
+   * Sets the path for processes to use for loading native libraries
+   * 
+   * @param nativePathItems
+   *          the nativePathItems to set
+   * @since 1.6.0
+   */
+  public void setNativeLibPaths(String... nativePathItems) {
+    this.nativePathItems = nativePathItems;
+  }
+
+  /**
    * Sets arbitrary configuration properties.
    * 
    * @since 1.6.0

http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
index 21c2bb7..54585fe 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.util.MonitorUtil;
 import org.apache.accumulo.minicluster.MiniAccumuloCluster;
 import org.apache.accumulo.minicluster.MiniAccumuloConfig;
@@ -41,7 +42,9 @@ public class ConfigurableMacIT extends AbstractMacIT {
   @Before
   public void setUp() throws Exception {
     MiniAccumuloConfig cfg = new MiniAccumuloConfig(createTestDir(this.getClass().getName()),
ROOT_PASSWORD);
+    cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath());
     configure(cfg);
+    cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString());
     configureForEnvironment(cfg, getClass(), createSharedTestDir(this.getClass().getName()
+ "-ssl"));
     cluster = new MiniAccumuloCluster(cfg);
     cluster.start();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 408d32c..5a79286 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@ -62,11 +62,15 @@ public class NativeMapIT {
     return new Value(String.format("r%09d", v).getBytes());
   }
 
-  @BeforeClass
-  public static void setUp() {
+  public static File nativeMapLocation() {
     File projectDir = new File(System.getProperty("user.dir")).getParentFile();
     File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION
+ "/accumulo-native-" + Constants.VERSION);
-    NativeMap.loadNativeLib(Collections.singletonList(nativeMapDir));
+    return nativeMapDir;
+  }
+
+  @BeforeClass
+  public static void setUp() {
+    NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
   }
 
   private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>>
iter) {


Mime
View raw message