accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1480705 - in /accumulo/trunk/server/src: main/java/org/apache/accumulo/server/mini/ test/java/org/apache/accumulo/server/mini/
Date Thu, 09 May 2013 16:01:33 GMT
Author: kturner
Date: Thu May  9 16:01:33 2013
New Revision: 1480705

URL: http://svn.apache.org/r1480705
Log:
ACCUMULO-1357 applied patch from Corey J Nolet for MiniAccumulo that enable remote java debugging
and allows users to configure memory size

Added:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java
Modified:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
    accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
    accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java

Added: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java?rev=1480705&view=auto
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java (added)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java Thu
May  9 16:01:33 2013
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+public enum MemoryUnit {
+  
+  BYTE(""), KILOBYTE("K"), MEGABYTE("M"), GIGABYTE("G"), TERABYTE("T");
+  
+  private final String printVal;
+  
+  private MemoryUnit(String printVal) {
+    this.printVal = printVal;
+  }
+  
+  public String getPrintVal(long memory) {
+    return memory + printVal;
+  }
+}

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
Thu May  9 16:01:33 2013
@@ -25,14 +25,19 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.Set;
 
+import org.apache.accumulo.core.util.Pair;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.server.master.Master;
 import org.apache.accumulo.server.tabletserver.TabletServer;
 import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
 import org.apache.accumulo.server.util.time.SimpleTimer;
 import org.apache.accumulo.start.Main;
 import org.apache.zookeeper.server.ZooKeeperServerMain;
@@ -98,6 +103,8 @@ public class MiniAccumuloCluster {
   private Process masterProcess;
   private Process[] tabletServerProcesses;
   
+  private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
+  
   private File zooCfgFile;
   
   private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
@@ -105,6 +112,10 @@ public class MiniAccumuloCluster {
   private MiniAccumuloConfig config;
   
   private Process exec(Class<? extends Object> clazz, String... args) throws IOException
{
+    return exec(clazz, Collections.singletonList("-Xmx" + config.getDefaultMemorySize()),
args);
+  }
+  
+  private Process exec(Class<? extends Object> clazz, List<String> extraJvmOpts,
String... args) throws IOException {
     String javaHome = System.getProperty("java.home");
     String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
     String classpath = System.getProperty("java.class.path");
@@ -114,10 +125,9 @@ public class MiniAccumuloCluster {
     String className = clazz.getCanonicalName();
     
     ArrayList<String> argList = new ArrayList<String>();
-    
-    argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC",
"-XX:CMSInitiatingOccupancyFraction=75",
-        Main.class.getName(), className));
-    
+    argList.addAll(Arrays.asList(javaBin, "-cp", classpath));
+    argList.addAll(extraJvmOpts);
+    argList.addAll(Arrays.asList("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
Main.class.getName(), className));
     argList.addAll(Arrays.asList(args));
     
     ProcessBuilder builder = new ProcessBuilder(argList);
@@ -146,6 +156,19 @@ public class MiniAccumuloCluster {
     return process;
   }
   
+  private Process exec(Class<? extends Object> clazz, ServerType serverType, String...
args) throws IOException {
+    
+    List<String> jvmOpts = new ArrayList<String>();
+    jvmOpts.add("-Xmx" + config.getMemoryConfig(serverType));
+    
+    if (config.isDebug()) {
+      Integer port = PortUtils.getRandomFreePort();
+      jvmOpts.addAll(buildRemoteDebugParams(port));
+      debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
+    }
+    return exec(clazz, jvmOpts, args);
+  }
+  
   /**
    * 
    * @param dir
@@ -232,7 +255,7 @@ public class MiniAccumuloCluster {
       }
     });
     
-    zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+    zooKeeperProcess = exec(Main.class, ServerType.ZOOKEEPER, ZooKeeperServerMain.class.getName(),
zooCfgFile.getAbsolutePath());
     
     // sleep a little bit to let zookeeper come up before calling init, seems to work better
     UtilWaitThread.sleep(250);
@@ -245,16 +268,28 @@ public class MiniAccumuloCluster {
     
     tabletServerProcesses = new Process[config.getNumTservers()];
     for (int i = 0; i < config.getNumTservers(); i++) {
-      tabletServerProcesses[i] = exec(TabletServer.class);
+      tabletServerProcesses[i] = exec(TabletServer.class, ServerType.TABLET_SERVER);
     }
     
-    masterProcess = exec(Master.class);
+    masterProcess = exec(Master.class, ServerType.MASTER);
+  }
+  
+  private List<String> buildRemoteDebugParams(int port) {
+    return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d",
port)});
   }
   
   /**
-   * @return Accumulo instance name
+   * @return generated remote debug ports if in debug mode.
+   * 
+   * @since 1.6.0
    */
+  public Set<Pair<ServerType,Integer>> getDebugPorts() {
+    return debugPorts;
+  }
   
+  /**
+   * @return Accumulo instance name
+   */
   public String getInstanceName() {
     return config.getInstanceName();
   }
@@ -262,7 +297,6 @@ public class MiniAccumuloCluster {
   /**
    * @return zookeeper connection string
    */
-  
   public String getZooKeepers() {
     return config.getZooKeepers();
   }
@@ -290,10 +324,8 @@ public class MiniAccumuloCluster {
   }
   
   /**
-   * 
    * @since 1.6.0
    */
-
   public MiniAccumuloConfig getConfig() {
     return config;
   }

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
Thu May  9 16:01:33 2013
@@ -36,6 +36,9 @@ public class MiniAccumuloConfig {
   private String rootPassword = null;
   private Map<String,String> siteConfig = new HashMap<String,String>();
   private int numTservers = 2;
+  private Map<ServerType,String> memoryConfig = new HashMap<ServerType,String>();
+  
+  private boolean debug = false;
   
   private String instanceName = "miniInstance";
   
@@ -47,12 +50,13 @@ public class MiniAccumuloConfig {
   private File walogDir;
   
   private Integer zooKeeperPort;
+  private String defaultMemorySize = "128M";
   
   private boolean initialized = false;
   
   /**
    * @param dir
-   *          An empty or nonexistant directoy that Accumulo and Zookeeper can store data
in. Creating the directory is left to the user. Java 7, Guava, and
+   *          An empty or nonexistant directory that Accumulo and Zookeeper can store data
in. Creating the directory is left to the user. Java 7, Guava, and
    *          Junit provide methods for creating temporary directories.
    * @param rootPassword
    *          The initial password for the Accumulo root user
@@ -108,9 +112,6 @@ public class MiniAccumuloConfig {
   
   /**
    * Set a given key/value on the site config if it doesn't already exist
-   * 
-   * @param key
-   * @param value
    */
   private void mergeProp(String key, String value) {
     if (!siteConfig.containsKey(key)) {
@@ -120,8 +121,6 @@ public class MiniAccumuloConfig {
   
   /**
    * Sets a given key with a random port for the value on the site config if it doesn't already
exist.
-   * 
-   * @param key
    */
   private void mergePropWithRandomPort(String key) {
     if (!siteConfig.containsKey(key)) {
@@ -177,6 +176,40 @@ public class MiniAccumuloConfig {
   }
   
   /**
+   * Sets the amount of memory to use in the master process. Calling this method is optional.
Default memory is 128M
+   * 
+   * @param serverType
+   *          the type of server to apply the memory settings
+   * @param memory
+   *          amount of memory to set
+   * 
+   * @param memoryUnit
+   *          the units for which to apply with the memory size
+   * 
+   * @since 1.6.0
+   */
+  public MiniAccumuloConfig setMemory(ServerType serverType, long memory, MemoryUnit memoryUnit)
{
+    this.memoryConfig.put(serverType, memoryUnit.getPrintVal(memory));
+    return this;
+  }
+  
+  /**
+   * Sets the default memory size to use. This value is also used when a ServerType has not
been configured explicitly. Calling this method is optional. Default
+   * memory is 128M
+   * 
+   * @param memory
+   *          amount of memory to set
+   * 
+   * @memoryUnit the units for which to apply with the memory size
+   * 
+   * @since 1.6.0
+   */
+  public MiniAccumuloConfig setDefaultMemorySize(long memory, MemoryUnit memoryUnit) {
+    this.defaultMemorySize = memoryUnit.getPrintVal(memory);
+    return this;
+  }
+  
+  /**
    * @return a copy of the site config
    */
   public Map<String,String> getSiteConfig() {
@@ -225,6 +258,14 @@ public class MiniAccumuloConfig {
     return walogDir;
   }
   
+  String getMemoryConfig(ServerType serverType) {
+    return memoryConfig.containsKey(serverType) ? memoryConfig.get(serverType) : defaultMemorySize;
+  }
+  
+  String getDefaultMemorySize() {
+    return defaultMemorySize;
+  }
+  
   /**
    * @return zookeeper connection string
    * 
@@ -234,16 +275,46 @@ public class MiniAccumuloConfig {
     return siteConfig.get(Property.INSTANCE_ZK_HOST.getKey());
   }
   
+  /**
+   * @return the base directory of the cluster configuration
+   */
   public File getDir() {
     return dir;
   }
   
+  /**
+   * @return the root password of this cluster configuration
+   */
   public String getRootPassword() {
     return rootPassword;
   }
   
+  /**
+   * @return the number of tservers configured for this cluster
+   */
   public int getNumTservers() {
     return numTservers;
   }
   
+  /**
+   * @return is the current configuration in debug mode?
+   * 
+   * @since 1.6.0
+   */
+  public boolean isDebug() {
+    return debug;
+  }
+  
+  /**
+   * @param debug
+   *          should the processes run remote debug servers?
+   * @return the current instance
+   * 
+   * @since 1.6.0
+   */
+  public MiniAccumuloConfig setDebug(boolean debug) {
+    this.debug = debug;
+    return this;
+  }
+  
 }

Added: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java?rev=1480705&view=auto
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java (added)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java Thu
May  9 16:01:33 2013
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+public enum ServerType {
+  MASTER, ZOOKEEPER, TABLET_SERVER
+}

Modified: accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
(original)
+++ accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
Thu May  9 16:01:33 2013
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.mini;
 import java.io.File;
 import java.util.Collections;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.UUID;
 
 import org.apache.accumulo.core.client.BatchWriter;
@@ -37,7 +38,7 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
+import org.apache.accumulo.core.util.Pair;
 import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -55,15 +56,12 @@ public class MiniAccumuloClusterTest {
   
   @BeforeClass
   public static void setupMiniCluster() throws Exception {
-    
-    folder.create();
-    
     Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
     
-    accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret");
-    
+    folder.create();
+    MiniAccumuloConfig config = new MiniAccumuloConfig(folder.getRoot(), "superSecret").setDebug(true);
+    accumulo = new MiniAccumuloCluster(config);
     accumulo.start();
-    
   }
   
   @Test(timeout = 30000)
@@ -184,10 +182,20 @@ public class MiniAccumuloClusterTest {
     conn.tableOperations().delete("table2");
   }
   
+  @Test(timeout = 10000)
+  public void testDebugPorts() {
+    
+    Set<Pair<ServerType,Integer>> debugPorts = accumulo.getDebugPorts();
+    Assert.assertEquals(4, debugPorts.size());
+    for (Pair<ServerType,Integer> debugPort : debugPorts) {
+      Assert.assertTrue(debugPort.getSecond() > 0);
+    }
+  }
+  
   @AfterClass
   public static void tearDownMiniCluster() throws Exception {
     accumulo.stop();
-    // folder.delete();
+    folder.delete();
   }
   
 }

Modified: accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java
(original)
+++ accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java
Thu May  9 16:01:33 2013
@@ -60,6 +60,16 @@ public class MiniAccumuloConfigTest {
     assertEquals("hdfs://", config.getSiteConfig().get(Property.INSTANCE_DFS_URI.getKey()));
   }
 
+  @Test
+  public void testMemoryConfig() {
+
+    MiniAccumuloConfig config = new MiniAccumuloConfig(tempFolder.getRoot(), "password").initialize();
+    config.setDefaultMemorySize(128, MemoryUnit.MEGABYTE);
+    assertEquals("128M", config.getMemoryConfig(ServerType.MASTER));
+    config.setMemory(ServerType.MASTER, 256, MemoryUnit.MEGABYTE);
+    assertEquals("256M", config.getMemoryConfig(ServerType.MASTER));
+  }
+
   @AfterClass
   public static void tearDown() {
     tempFolder.delete();



Mime
View raw message