accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1432275 - in /accumulo/trunk: examples/instamo/src/main/java/org/apache/accumulo/instamo/ examples/instamo/src/test/java/org/apache/accumulo/instamo/ server/src/main/c++/nativeMap/ test/src/main/java/org/apache/accumulo/test/ test/src/test...
Date Fri, 11 Jan 2013 20:38:18 GMT
Author: kturner
Date: Fri Jan 11 20:38:18 2013
New Revision: 1432275

URL: http://svn.apache.org/viewvc?rev=1432275&view=rev
Log:
ACCUMULO-880 made accumulo mini cluster support running multiple tablet server and defaulted
to running two

Added:
    accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MacConfig.java
Modified:
    accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
    accumulo/trunk/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java
    accumulo/trunk/server/src/main/c++/nativeMap/   (props changed)
    accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
    accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java

Modified: accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java?rev=1432275&r1=1432274&r2=1432275&view=diff
==============================================================================
--- accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
(original)
+++ accumulo/trunk/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
Fri Jan 11 20:38:18 2013
@@ -17,11 +17,11 @@
 package org.apache.accumulo.instamo;
 
 import java.io.File;
-import java.util.HashMap;
 import java.util.UUID;
 
 import org.apache.accumulo.server.test.continuous.ContinuousIngest;
 import org.apache.accumulo.server.test.continuous.ContinuousVerify;
+import org.apache.accumulo.test.MacConfig;
 import org.apache.accumulo.test.MiniAccumuloCluster;
 import org.apache.commons.io.FileUtils;
 
@@ -52,7 +52,7 @@ public class MapReduceExample {
     File tmpDir = new File(FileUtils.getTempDirectory(), "macc-" + UUID.randomUUID().toString());
     
     try {
-      MiniAccumuloCluster la = new MiniAccumuloCluster(tmpDir, "pass1234", new HashMap<String,String>());
+      MiniAccumuloCluster la = new MiniAccumuloCluster(new MacConfig(tmpDir, "pass1234"));
       la.start();
       
       System.out.println("\n   ---- Running Mapred Against Accumulo\n");

Modified: accumulo/trunk/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java?rev=1432275&r1=1432274&r2=1432275&view=diff
==============================================================================
--- accumulo/trunk/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java
(original)
+++ accumulo/trunk/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java
Fri Jan 11 20:38:18 2013
@@ -17,9 +17,7 @@ package org.apache.accumulo.instamo;
  */
 
 
-import java.util.HashMap;
-
-import org.apache.accumulo.instamo.AccumuloApp;
+import org.apache.accumulo.test.MacConfig;
 import org.apache.accumulo.test.MiniAccumuloCluster;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -41,7 +39,7 @@ public class ExampleAccumuloUnitTest {
 
     folder.create();
     
-    accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret", new HashMap<String,String>());
+    accumulo = new MiniAccumuloCluster(new MacConfig(folder.getRoot(), "superSecret"));
     
     accumulo.start();
     

Propchange: accumulo/trunk/server/src/main/c++/nativeMap/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Fri Jan 11 20:38:18 2013
@@ -1 +1,2 @@
 *.jnilib
+libNativeMap-Linux-amd64-64.so

Added: accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MacConfig.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MacConfig.java?rev=1432275&view=auto
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MacConfig.java (added)
+++ accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MacConfig.java Fri Jan 11 20:38:18
2013
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed
to constructor and all other configurations are optional.
+ */
+
+public class MacConfig {
+  
+  private File dir = null;
+  private String rootPassword = null;
+  private Map<String,String> siteConfig = Collections.emptyMap();
+  private int numTservers = 2;
+  
+  /**
+   * @param dir
+   *          A local directory where mini accumulo cluster will store all of its data. This
directory should be empty or non-existant.
+   * @param rootPassword
+   *          The initial password for the Accumulo root user
+   */
+
+  public MacConfig(File dir, String rootPassword) {
+    this.dir = dir;
+    this.rootPassword = rootPassword;
+  }
+  
+  public File getDir() {
+    return dir;
+  }
+  
+  public String getRootPassword() {
+    return rootPassword;
+  }
+  
+  public int getNumTservers() {
+    return numTservers;
+  }
+
+  /**
+   * Calling this method is optional. If not set, it defaults to two.
+   * 
+   * @param numTservers
+   *          the number of tablet servers that mini accumulo cluster should start
+   * @return
+   */
+  
+  public MacConfig setNumTservers(int numTservers) {
+    if (numTservers < 1)
+      throw new IllegalArgumentException("Must have at least one tablet server");
+    this.numTservers = numTservers;
+    return this;
+  }
+
+  public Map<String,String> getSiteConfig() {
+    return siteConfig;
+  }
+
+  
+  /**
+   * Calling this method is optional. If not set, it defautls to an empty map.
+   * 
+   * @param siteConfig
+   *          key/values that you normally put in accumulo-site.xml can be put here
+   * @return
+   */
+  
+  public MacConfig setSiteConfig(Map<String,String> siteConfig) {
+    this.siteConfig = siteConfig;
+    return this;
+  }
+}
\ No newline at end of file

Modified: accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java?rev=1432275&r1=1432274&r2=1432275&view=diff
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java (original)
+++ accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java Fri
Jan 11 20:38:18 2013
@@ -26,6 +26,7 @@ import java.io.InputStreamReader;
 import java.net.ServerSocket;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -102,7 +103,6 @@ public class MiniAccumuloCluster {
     }
   }
 
-  private File baseDir;
   private File libDir;
   private File confDir;
   private File zooKeeperDir;
@@ -113,15 +113,13 @@ public class MiniAccumuloCluster {
   
   private Process zooKeeperProcess;
   private Process masterProcess;
-  private Process tabletServerProcess;
-  private Process loggerProcess;
 
   private int zooKeeperPort;
   
   private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
-  private String rootPassword;
-
 
+  private MacConfig config;
+  private Process[] tabletServerProcesses;
 
   private int getRandomFreePort() {
     Random r = new Random();
@@ -166,7 +164,7 @@ public class MiniAccumuloCluster {
     
     ProcessBuilder builder = new ProcessBuilder(argList);
     
-    builder.environment().put("ACCUMULO_HOME", baseDir.getAbsolutePath());
+    builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
     builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
 
     Process process = builder.start();
@@ -204,23 +202,22 @@ public class MiniAccumuloCluster {
    * @throws IOException
    */
 
-  public MiniAccumuloCluster(File dir, String rootPassword, Map<String,String> siteConfig)
throws IOException {
+  public MiniAccumuloCluster(MacConfig config) throws IOException {
 
-    if (dir.exists() && !dir.isDirectory())
-      throw new IllegalArgumentException("Must pass in directory, " + dir + " is a file");
+    if (config.getDir().exists() && !config.getDir().isDirectory())
+      throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + "
is a file");
     
-    if (dir.exists() && dir.list().length != 0)
-      throw new IllegalArgumentException("Directory " + dir + " is not empty");
+    if (config.getDir().exists() && config.getDir().list().length != 0)
+      throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
     
-    this.rootPassword = rootPassword;
-
-    baseDir = dir;
-    libDir = new File(dir, "lib");
-    confDir = new File(dir, "conf");
-    accumuloDir = new File(dir, "accumulo");
-    zooKeeperDir = new File(dir, "zookeeper");
-    logDir = new File(dir, "logs");
-    walogDir = new File(dir, "walogs");
+    this.config = config;
+
+    libDir = new File(config.getDir(), "lib");
+    confDir = new File(config.getDir(), "conf");
+    accumuloDir = new File(config.getDir(), "accumulo");
+    zooKeeperDir = new File(config.getDir(), "zookeeper");
+    logDir = new File(config.getDir(), "logs");
+    walogDir = new File(config.getDir(), "walogs");
     
     confDir.mkdirs();
     accumuloDir.mkdirs();
@@ -236,12 +233,15 @@ public class MiniAccumuloCluster {
     FileWriter fileWriter = new FileWriter(siteFile);
     fileWriter.append("<configuration>\n");
     
+    HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
     appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
     appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
     appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
     appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
     appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + getRandomFreePort(), siteConfig);
     appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + getRandomFreePort(), siteConfig);
+    appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
     appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
     appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
     appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
@@ -305,13 +305,17 @@ public class MiniAccumuloCluster {
     // sleep a little bit to let zookeeper come up before calling init, seems to work better
     UtilWaitThread.sleep(250);
 
-    Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password",
rootPassword);
+    Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password",
config.getRootPassword());
     int ret = initProcess.waitFor();
     if (ret != 0) {
       throw new RuntimeException("Initialize process returned " + ret);
     }
-   
-    tabletServerProcess = exec(TabletServer.class);
+    
+    tabletServerProcesses = new Process[config.getNumTservers()];
+    for (int i = 0; i < config.getNumTservers(); i++) {
+      tabletServerProcesses[i] = exec(TabletServer.class);
+    }
+
     masterProcess = exec(Master.class);
   }
 
@@ -344,10 +348,11 @@ public class MiniAccumuloCluster {
       zooKeeperProcess.destroy();
     if (masterProcess != null)
       masterProcess.destroy();
-    if (tabletServerProcess != null)
-      tabletServerProcess.destroy();
-    if (loggerProcess != null)
-      loggerProcess.destroy();
+    if (tabletServerProcesses != null) {
+      for (Process tserver : tabletServerProcesses) {
+        tserver.destroy();
+      }
+    }
     
     for (LogWriter lw : logWriters)
       lw.flush();

Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java?rev=1432275&r1=1432274&r2=1432275&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
(original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java
Fri Jan 11 20:38:18 2013
@@ -18,7 +18,6 @@ package org.apache.accumulo.test;
 
 import java.io.File;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map.Entry;
 import java.util.UUID;
 
@@ -60,7 +59,7 @@ public class MiniAccumuloClusterTest {
     
     Logger.getLogger("org.apache.zookeeper").setLevel(Level.WARN);
 
-    accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret", new HashMap<String,String>());
+    accumulo = new MiniAccumuloCluster(new MacConfig(folder.getRoot(), "superSecret").setNumTservers(2));
     
     accumulo.start();
     



Mime
View raw message