accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject svn commit: r1433166 [18/20] - in /accumulo/branches/ACCUMULO-259: ./ assemble/ assemble/platform/ assemble/scripts/ assemble/scripts/init.d/ bin/ conf/examples/1GB/native-standalone/ conf/examples/1GB/standalone/ conf/examples/2GB/native-standalone/ c...
Date Mon, 14 Jan 2013 22:03:34 GMT
Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java Mon Jan 14 22:03:24 2013
@@ -25,7 +25,7 @@ import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.Socket;
 import java.net.URLEncoder;
-import java.nio.charset.Charset;
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
@@ -35,12 +35,7 @@ import java.util.regex.Pattern;
 
 import javax.net.SocketFactory;
 
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.MissingOptionException;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
+import org.apache.accumulo.core.cli.Help;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.commons.lang.math.LongRange;
 import org.apache.log4j.Category;
@@ -53,6 +48,9 @@ import org.apache.log4j.spi.ThrowableInf
 import org.apache.log4j.varia.LevelRangeFilter;
 import org.apache.log4j.xml.XMLLayout;
 
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.Parameter;
+
 public class SendLogToChainsaw extends XMLLayout {
   
   private static Pattern logPattern = Pattern.compile(
@@ -71,8 +69,6 @@ public class SendLogToChainsaw extends X
   private LongRange dateFilter = null;
   
   private LevelRangeFilter levelFilter = null;
-
-  private static final Charset utf8 = Charset.forName("UTF8");
   
   public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
     
@@ -138,7 +134,7 @@ public class SendLogToChainsaw extends X
                 out = convertLine(line, threadName);
                 if (null != out) {
                   if (socket != null && socket.isConnected())
-                    socket.getOutputStream().write(out.getBytes(utf8));
+                    socket.getOutputStream().write(out.getBytes());
                   else
                     System.err.println("Unable to send data to transport");
                 }
@@ -214,53 +210,49 @@ public class SendLogToChainsaw extends X
     return result;
   }
   
-  private static Options getOptions() {
-    Options opts = new Options();
+  private static class DateConverter implements IStringConverter<Date> {
+    @Override
+    public Date convert(String value) {
+      SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
+      try {
+        return formatter.parse(value);
+      } catch (ParseException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    
+  }
+  
+  private static class Opts extends Help {
+    
+    @Parameter(names={"-d", "--logDirectory"}, description="ACCUMULO log directory path", required=true)
+    String dir;
+    
+    @Parameter(names={"-f", "--fileFilter"}, description="filter to apply to names of logs")
+    String filter;
+    
+    @Parameter(names={"-h", "--host"}, description="host where chainsaw is running", required=true)
+    String hostname;
+    
+    @Parameter(names={"-p", "--port"}, description="port where XMLSocketReceiver is listening", required=true)
+    int portnum;
+    
+    @Parameter(names={"-s", "--start"}, description="start date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+    Date startDate;
     
-    Option dirOption = new Option("d", "logDirectory", true, "ACCUMULO log directory path");
-    dirOption.setArgName("dir");
-    dirOption.setRequired(true);
-    opts.addOption(dirOption);
-    
-    Option fileFilterOption = new Option("f", "fileFilter", true, "filter to apply to names of logs");
-    fileFilterOption.setArgName("filter");
-    fileFilterOption.setRequired(false);
-    opts.addOption(fileFilterOption);
-    
-    Option hostOption = new Option("h", "host", true, "host where chainsaw is running");
-    hostOption.setArgName("hostname");
-    hostOption.setRequired(true);
-    opts.addOption(hostOption);
-    
-    Option portOption = new Option("p", "port", true, "port where XMLSocketReceiver is listening");
-    portOption.setArgName("portnum");
-    portOption.setRequired(true);
-    opts.addOption(portOption);
-    
-    Option startOption = new Option("s", "start", true, "start date filter (yyyyMMddHHmmss)");
-    startOption.setArgName("date");
-    startOption.setRequired(true);
-    opts.addOption(startOption);
-    
-    Option endOption = new Option("e", "end", true, "end date filter (yyyyMMddHHmmss)");
-    endOption.setArgName("date");
-    endOption.setRequired(true);
-    opts.addOption(endOption);
-    
-    Option levelOption = new Option("l", "level", true, "filter log level");
-    levelOption.setArgName("level");
-    levelOption.setRequired(false);
-    opts.addOption(levelOption);
-    
-    Option msgFilter = new Option("m", "messageFilter", true, "regex filter for log messages");
-    msgFilter.setArgName("regex");
-    msgFilter.setRequired(false);
-    opts.addOption(msgFilter);
+    @Parameter(names={"-e", "--end"}, description="end date filter (yyyyMMddHHmmss)", required=true, converter=DateConverter.class)
+    Date endDate;
     
-    return opts;
+    @Parameter(names={"-l", "--level"}, description="filter log level")
+    String level;
     
+    @Parameter(names={"-m", "--messageFilter"}, description="regex filter for log messages")
+    String regex;
   }
   
+  
+  
+  
   /**
    * 
    * @param args
@@ -275,47 +267,10 @@ public class SendLogToChainsaw extends X
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
+    Opts opts = new Opts();
+    opts.parseArgs(SendLogToChainsaw.class.getName(), args);
     
-    Options o = getOptions();
-    CommandLine cl = null;
-    try {
-    cl = new BasicParser().parse(o, args);
-    } catch (MissingOptionException e) {
-    	System.out.println(e.toString());
-    	HelpFormatter formatter = new HelpFormatter();
-    	formatter.printHelp( "SendLogToChainsaw", o );
-    	return;
-    }
-    
-    String logDir = cl.getOptionValue(o.getOption("d").getOpt());
-    String fileNameFilter = null;
-    if (cl.hasOption(o.getOption("f").getOpt()))
-      fileNameFilter = cl.getOptionValue(o.getOption("f").getOpt());
-    String chainsawHost = cl.getOptionValue(o.getOption("h").getOpt());
-    int chainsawPort = 0;
-    try {
-      chainsawPort = Integer.parseInt(cl.getOptionValue(o.getOption("p").getOpt()));
-    } catch (NumberFormatException nfe) {
-      System.err.println("Unable to parse port number");
-      System.exit(-1);
-    }
-    SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
-    Date startDate = null;
-    if (cl.hasOption(o.getOption("s").getOpt())) {
-      startDate = formatter.parse(cl.getOptionValue(o.getOption("s").getOpt()));
-    }
-    Date endDate = null;
-    if (cl.hasOption(o.getOption("e").getOpt())) {
-      endDate = formatter.parse(cl.getOptionValue(o.getOption("e").getOpt()));
-    }
-    String msgFilter = null;
-    if (cl.hasOption(o.getOption("m").getOpt()))
-      msgFilter = cl.getOptionValue(o.getOption("m").getOpt());
-    String levelFilter = null;
-    if (cl.hasOption(o.getOption("l").getOpt()))
-      levelFilter = cl.getOptionValue(o.getOption("l").getOpt());
-    
-    SendLogToChainsaw c = new SendLogToChainsaw(logDir, fileNameFilter, chainsawHost, chainsawPort, startDate, endDate, msgFilter, levelFilter);
+    SendLogToChainsaw c = new SendLogToChainsaw(opts.dir, opts.filter, opts.hostname, opts.portnum, opts.startDate, opts.endDate, opts.regex, opts.level);
     c.processLogFiles();
   }
   

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java Mon Jan 14 22:03:24 2013
@@ -16,8 +16,6 @@
  */
 package org.apache.accumulo.server.util;
 
-import java.nio.charset.Charset;
-
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -28,9 +26,6 @@ import org.apache.accumulo.server.zookee
 import org.apache.zookeeper.KeeperException;
 
 public class SystemPropUtil {
-
-  private static final Charset utf8 = Charset.forName("UTF8");
-	
   public static boolean setSystemProperty(String property, String value) throws KeeperException, InterruptedException {
     Property p = Property.getPropertyByKey(property);
     if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property))
@@ -38,7 +33,7 @@ public class SystemPropUtil {
     
     // create the zk node for this property and set it's data to the specified value
     String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
-    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(utf8), NodeExistsPolicy.OVERWRITE);
+    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(), NodeExistsPolicy.OVERWRITE);
     
     return true;
   }

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java Mon Jan 14 22:03:24 2013
@@ -53,7 +53,7 @@ import org.apache.thrift.transport.TTran
 public class TServerUtils {
   private static final Logger log = Logger.getLogger(TServerUtils.class);
   
-  public static ThreadLocal<String> clientAddress = new ThreadLocal<String>();
+  public static final ThreadLocal<String> clientAddress = new ThreadLocal<String>();
   
   public static class ServerPort {
     public final TServer server;
@@ -85,7 +85,9 @@ public class TServerUtils {
    */
   public static ServerPort startServer(AccumuloConfiguration conf, Property portHintProperty, TProcessor processor, String serverName, String threadName,
       Property portSearchProperty,
-      Property minThreadProperty, Property timeBetweenThreadChecksProperty) throws UnknownHostException {
+      Property minThreadProperty, 
+      Property timeBetweenThreadChecksProperty, 
+      Property maxMessageSizeProperty) throws UnknownHostException {
     int portHint = conf.getPort(portHintProperty);
     int minThreads = 2;
     if (minThreadProperty != null)
@@ -93,6 +95,9 @@ public class TServerUtils {
     long timeBetweenThreadChecks = 1000;
     if (timeBetweenThreadChecksProperty != null)
       timeBetweenThreadChecks = conf.getTimeInMillis(timeBetweenThreadChecksProperty);
+    long maxMessageSize = 10 * 1000 * 1000;
+    if (maxMessageSizeProperty != null)
+      maxMessageSize = conf.getMemoryInBytes(maxMessageSizeProperty);
     boolean portSearch = false;
     if (portSearchProperty != null)
       portSearch = conf.getBoolean(portSearchProperty);
@@ -111,7 +116,7 @@ public class TServerUtils {
         if (port > 65535)
           port = 1024 + port % (65535 - 1024);
         try {
-          return TServerUtils.startTServer(port, processor, serverName, threadName, minThreads, timeBetweenThreadChecks);
+          return TServerUtils.startTServer(port, processor, serverName, threadName, minThreads, timeBetweenThreadChecks, maxMessageSize);
         } catch (Exception ex) {
           log.info("Unable to use port " + port + ", retrying. (Thread Name = " + threadName + ")");
           UtilWaitThread.sleep(250);
@@ -178,7 +183,7 @@ public class TServerUtils {
   }
   
   public static ServerPort startHsHaServer(int port, TProcessor processor, final String serverName, String threadName, final int numThreads,
-      long timeBetweenThreadChecks) throws TTransportException {
+      long timeBetweenThreadChecks, long maxMessageSize) throws TTransportException {
     TNonblockingServerSocket transport = new TNonblockingServerSocket(port);
     THsHaServer.Args options = new THsHaServer.Args(transport);
     options.protocolFactory(ThriftUtil.protocolFactory());
@@ -192,7 +197,7 @@ public class TServerUtils {
       @Override
       public void run() {
         if (pool.getCorePoolSize() <= pool.getActiveCount()) {
-          int larger = pool.getCorePoolSize() + 2;
+          int larger = pool.getCorePoolSize() + Math.min(pool.getQueue().size(), 2);
           log.info("Increasing server thread pool size on " + serverName + " to " + larger);
           pool.setMaximumPoolSize(larger);
           pool.setCorePoolSize(larger);
@@ -239,9 +244,9 @@ public class TServerUtils {
     return new ServerPort(new TThreadPoolServer(options), port);
   }
   
-  public static ServerPort startTServer(int port, TProcessor processor, String serverName, String threadName, int numThreads, long timeBetweenThreadChecks)
+  public static ServerPort startTServer(int port, TProcessor processor, String serverName, String threadName, int numThreads, long timeBetweenThreadChecks, long maxMessageSize)
       throws TTransportException {
-    ServerPort result = startHsHaServer(port, processor, serverName, threadName, numThreads, timeBetweenThreadChecks);
+    ServerPort result = startHsHaServer(port, processor, serverName, threadName, numThreads, timeBetweenThreadChecks, maxMessageSize);
     // ServerPort result = startThreadPoolServer(port, processor, serverName, threadName, -1);
     final TServer finalServer = result.server;
     Runnable serveTask = new Runnable() {

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java Mon Jan 14 22:03:24 2013
@@ -16,28 +16,33 @@
  */
 package org.apache.accumulo.server.util;
 
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
 
+import org.apache.accumulo.server.cli.ClientOpts;
 import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 
+import com.beust.jcommander.Parameter;
+
 public class TableDiskUsage {
   
+  static class Opts extends ClientOpts {
+    @Parameter(description=" <table> { <table> ... } ")
+    List<String> tables = new ArrayList<String>();
+  }
+  
   /**
    * @param args
    */
   public static void main(String[] args) throws Exception {
     FileSystem fs = FileSystem.get(new Configuration());
-    
-    Instance instance = HdfsZooInstance.getInstance();
-    ServerConfiguration conf = new ServerConfiguration(instance);
-    Connector conn = instance.getConnector("root", "secret");
-    
-    org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(conf.getConfiguration(), Arrays.asList(args), fs, conn);
+    Opts opts = new Opts();
+    opts.parseArgs(TableDiskUsage.class.getName(), args);
+    Connector conn = opts.getConnector();
+    org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn);
   }
   
 }

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java Mon Jan 14 22:03:24 2013
@@ -16,8 +16,6 @@
  */
 package org.apache.accumulo.server.util;
 
-import java.nio.charset.Charset;
-
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -28,9 +26,6 @@ import org.apache.accumulo.server.zookee
 import org.apache.zookeeper.KeeperException;
 
 public class TablePropUtil {
-
-  private static final Charset utf8 = Charset.forName("UTF8");
-  
   public static boolean setTableProperty(String tableId, String property, String value) throws KeeperException, InterruptedException {
     if (!isPropertyValid(property, value))
       return false;
@@ -41,7 +36,7 @@ public class TablePropUtil {
     
     // create the zk node for this property and set it's data to the specified value
     String zPath = zkTablePath + "/" + property;
-    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(utf8), NodeExistsPolicy.OVERWRITE);
+    ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(), NodeExistsPolicy.OVERWRITE);
     
     return true;
   }

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java Mon Jan 14 22:03:24 2013
@@ -19,22 +19,33 @@ package org.apache.accumulo.server.util;
 import java.util.List;
 
 import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.zookeeper.ZooLock;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 
+import com.beust.jcommander.Parameter;
+
 public class TabletServerLocks {
   
+  static class Opts extends Help {
+    @Parameter(names="-list")
+    boolean list = false;
+    @Parameter(names="-delete")
+    String delete = null;
+  }
   /**
    * @param args
    */
   public static void main(String[] args) throws Exception {
     
     String tserverPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTSERVERS;
+    Opts opts = new Opts();
+    opts.parseArgs(TabletServerLocks.class.getName(), args);
     
-    if (args.length == 1 && args[0].equals("-list")) {
+    if (opts.list) {
       IZooReaderWriter zoo = ZooReaderWriter.getInstance();
       
       List<String> tabletServers = zoo.getChildren(tserverPath);
@@ -48,7 +59,7 @@ public class TabletServerLocks {
         
         System.out.printf("%32s %16s%n", tabletServer, holder);
       }
-    } else if (args.length == 2 && args[0].equals("-delete")) {
+    } else if (opts.delete != null) {
       ZooLock.deleteLock(tserverPath + "/" + args[1]);
     } else {
       System.out.println("Usage : " + TabletServerLocks.class.getName() + " -list|-delete <tserver lock>");

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java Mon Jan 14 22:03:24 2013
@@ -16,9 +16,7 @@
  */
 package org.apache.accumulo.server.util;
 
-import java.io.PrintWriter;
 import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -32,17 +30,15 @@ import java.util.concurrent.ExecutorServ
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-import jline.ConsoleReader;
-
 import org.apache.accumulo.cloudtrace.instrument.Tracer;
 import org.apache.accumulo.cloudtrace.thrift.TInfo;
 import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Range;
@@ -59,107 +55,52 @@ import org.apache.accumulo.core.tabletse
 import org.apache.accumulo.core.util.ThriftUtil;
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.MissingArgumentException;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
 import org.apache.hadoop.io.Text;
 import org.apache.thrift.TException;
 import org.apache.thrift.TServiceClient;
 
-public class VerifyTabletAssignments {
+import com.beust.jcommander.Parameter;
 
-  private static final Charset utf8 = Charset.forName("UTF8");
+public class VerifyTabletAssignments {
+  
+  static class Opts extends ClientOpts {
+    @Parameter(names={"-v", "--verbose"}, description="verbose mode (prints locations of tablets)")
+    boolean verbose = false;
+  }
   
   public static void main(String[] args) throws Exception {
-    Options opts = new Options();
-    
-    Option zooKeeperInstance = new Option("z", "zooKeeperInstance", true, "use a zookeeper instance with the given instance name and list of zoo hosts");
-    zooKeeperInstance.setArgName("name hosts");
-    zooKeeperInstance.setArgs(2);
-    opts.addOption(zooKeeperInstance);
-    
-    Option usernameOption = new Option("u", "user", true, "username (required)");
-    usernameOption.setArgName("user");
-    usernameOption.setRequired(true);
-    opts.addOption(usernameOption);
-    
-    Option passwOption = new Option("p", "password", true, "password (prompt for password if this option is missing)");
-    passwOption.setArgName("pass");
-    opts.addOption(passwOption);
-    
-    Option verboseOption = new Option("v", "verbose", false, "verbose mode (prints locations of tablets)");
-    opts.addOption(verboseOption);
-    
-    CommandLine cl = null;
-    String user = null;
-    String passw = null;
-    Instance instance = null;
-    ConsoleReader reader = new ConsoleReader();
-    try {
-      cl = new BasicParser().parse(opts, args);
-      
-      if (cl.hasOption(zooKeeperInstance.getOpt()) && cl.getOptionValues(zooKeeperInstance.getOpt()).length != 2)
-        throw new MissingArgumentException(zooKeeperInstance);
-      
-      user = cl.getOptionValue(usernameOption.getOpt());
-      passw = cl.getOptionValue(passwOption.getOpt());
-      
-      if (cl.hasOption(zooKeeperInstance.getOpt())) {
-        String[] zkOpts = cl.getOptionValues(zooKeeperInstance.getOpt());
-        instance = new ZooKeeperInstance(zkOpts[0], zkOpts[1]);
-      } else {
-        instance = HdfsZooInstance.getInstance();
-      }
-      
-      if (passw == null)
-        passw = reader.readLine("Enter current password for '" + user + "'@'" + instance.getInstanceName() + "': ", '*');
-      if (passw == null) {
-        reader.printNewline();
-        return;
-      } // user canceled
-      
-      if (cl.getArgs().length != 0)
-        throw new ParseException("Unrecognized arguments: " + cl.getArgList());
-      
-    } catch (ParseException e) {
-      PrintWriter pw = new PrintWriter(System.err);
-      new HelpFormatter().printHelp(pw, Integer.MAX_VALUE, "accumulo " + VerifyTabletAssignments.class.getName(), null, opts, 2, 5, null, true);
-      pw.flush();
-      System.exit(1);
-    }
-    
-    Connector conn = instance.getConnector(user, passw.getBytes(utf8));
-    ServerConfiguration conf = new ServerConfiguration(instance);
+    Opts opts = new Opts();
+    opts.parseArgs(VerifyTabletAssignments.class.getName(), args);
+
+    Connector conn = opts.getConnector();
     for (String table : conn.tableOperations().list())
-      checkTable(conf.getConfiguration(), user, passw, table, null, cl.hasOption(verboseOption.getOpt()));
+      checkTable(opts, table, null);
     
   }
   
-  private static void checkTable(final AccumuloConfiguration conf, final String user, final String pass, String table, HashSet<KeyExtent> check, boolean verbose)
+  private static void checkTable(final Opts opts, String tableName, HashSet<KeyExtent> check)
       throws AccumuloException,
       AccumuloSecurityException, TableNotFoundException, InterruptedException {
     
     if (check == null)
-      System.out.println("Checking table " + table);
+      System.out.println("Checking table " + tableName);
     else
-      System.out.println("Checking table " + table + " again, failures " + check.size());
+      System.out.println("Checking table " + tableName + " again, failures " + check.size());
     
     Map<KeyExtent,String> locations = new TreeMap<KeyExtent,String>();
     SortedSet<KeyExtent> tablets = new TreeSet<KeyExtent>();
     
-    MetadataTable.getEntries(HdfsZooInstance.getInstance(),
-        new AuthInfo(user, ByteBuffer.wrap(pass.getBytes(utf8)), HdfsZooInstance.getInstance().getInstanceID()), table, false, locations, tablets);
+    Connector conn = opts.getConnector();
+    Instance inst = conn.getInstance();
+    MetadataTable.getEntries(conn.getInstance(),
+        new AuthInfo(opts.user, ByteBuffer.wrap(opts.getPassword()), inst.getInstanceID()), tableName, false, locations, tablets);
     
     final HashSet<KeyExtent> failures = new HashSet<KeyExtent>();
     
     for (KeyExtent keyExtent : tablets)
       if (!locations.containsKey(keyExtent))
         System.out.println(" Tablet " + keyExtent + " has no location");
-      else if (verbose)
+      else if (opts.verbose)
         System.out.println(" Tablet " + keyExtent + " is located at " + locations.get(keyExtent));
     
     Map<String,List<KeyExtent>> extentsPerServer = new TreeMap<String,List<KeyExtent>>();
@@ -176,14 +117,14 @@ public class VerifyTabletAssignments {
     }
     
     ExecutorService tp = Executors.newFixedThreadPool(20);
-    
+    final ServerConfiguration conf = new ServerConfiguration(inst);
     for (final Entry<String,List<KeyExtent>> entry : extentsPerServer.entrySet()) {
       Runnable r = new Runnable() {
         
         @Override
         public void run() {
           try {
-            checkTabletServer(conf, user, ByteBuffer.wrap(pass.getBytes(utf8)), entry, failures);
+            checkTabletServer(conf.getConfiguration(), opts.user, ByteBuffer.wrap(opts.getPassword()), entry, failures);
           } catch (Exception e) {
             System.err.println("Failure on ts " + entry.getKey() + " " + e.getMessage());
             e.printStackTrace();
@@ -201,7 +142,7 @@ public class VerifyTabletAssignments {
     while (!tp.awaitTermination(1, TimeUnit.HOURS)) {}
     
     if (failures.size() > 0)
-      checkTable(conf, user, pass, table, failures, verbose);
+      checkTable(opts, tableName, failures);
   }
   
   private static void checkFailures(String server, HashSet<KeyExtent> failures, MultiScanResult scanResult) {

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/ZooZap.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/ZooZap.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/ZooZap.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/ZooZap.java Mon Jan 14 22:03:24 2013
@@ -19,12 +19,15 @@ package org.apache.accumulo.server.util;
 import java.util.List;
 
 import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.server.cli.ClientOpts;
 import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
 import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
-import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.zookeeper.ZooLock;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
 public class ZooZap {
   
   static boolean verbose = false;
@@ -37,49 +40,44 @@ public class ZooZap {
       System.out.println(msg);
   }
   
-  public static void main(String[] args) {
-    
+  static class Opts extends ClientOpts {
+    @Parameter(names="-master", description="remove master locks")
     boolean zapMaster = false;
+    @Parameter(names="-tservers", description="remove tablet server locks")
     boolean zapTservers = false;
+    @Parameter(names="-tracers", description="remove tracer locks")
     boolean zapTracers = false;
+    @Parameter(names="-verbose", description="print out messages about progress")
+    boolean verbose = false;
+  }
+  
+  public static void main(String[] args) {
+    Opts opts = new Opts();
+    opts.parseArgs(ZooZap.class.getName(), args);
     
-    if (args.length == 0) {
-      printUsage();
-      return;
-    }
-    
-    for (int i = 0; i < args.length; i++) {
-      if (args[i].equals("-tservers")) {
-        zapTservers = true;
-      } else if (args[i].equals("-master")) {
-        zapMaster = true;
-      } else if (args[i].equals("-tracers")) {
-        zapTracers = true;
-      } else if (args[i].equals("-verbose")) {
-        verbose = true;
-      } else {
-        printUsage();
+    if (!opts.zapMaster && !opts.zapTservers && !opts.zapTracers)
+    {
+        new JCommander(opts).usage();
         return;
-      }
     }
     
+    String iid = opts.getInstance().getInstanceID();
     IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-    String iid = HdfsZooInstance.getInstance().getInstanceID();
     
-    if (zapMaster) {
+    if (opts.zapMaster) {
       String masterLockPath = Constants.ZROOT + "/" + iid + Constants.ZMASTER_LOCK;
       
       zapDirectory(zoo, masterLockPath);
     }
     
-    if (zapTservers) {
+    if (opts.zapTservers) {
       String tserversPath = Constants.ZROOT + "/" + iid + Constants.ZTSERVERS;
       try {
         List<String> children = zoo.getChildren(tserversPath);
         for (String child : children) {
           message("Deleting " + tserversPath + "/" + child + " from zookeeper");
           
-          if (zapMaster)
+          if (opts.zapMaster)
             ZooReaderWriter.getInstance().recursiveDelete(tserversPath + "/" + child, NodeMissingPolicy.SKIP);
           else {
             String path = tserversPath + "/" + child;
@@ -95,7 +93,7 @@ public class ZooZap {
       }
     }
     
-    if (zapTracers) {
+    if (opts.zapTracers) {
       String path = Constants.ZROOT + "/" + iid + Constants.ZTRACERS;
       zapDirectory(zoo, path);
     }
@@ -113,9 +111,4 @@ public class ZooZap {
       e.printStackTrace();
     }
   }
-  
-  private static void printUsage() {
-    System.err.println("Usage : " + ZooZap.class.getName() + " [-verbose] [-tservers] [-master] [-tracers]");
-  }
-  
 }

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java Mon Jan 14 22:03:24 2013
@@ -33,7 +33,9 @@ public class TransactionWatcher extends 
     
     @Override
     public boolean transactionAlive(String type, long tid) throws Exception {
-      return rdr.exists(ZooUtil.getRoot(instance) + "/" + type + "/" + Long.toString(tid));
+      String path = ZooUtil.getRoot(instance) + "/" + type + "/" + Long.toString(tid);
+      rdr.sync(path);
+      return rdr.exists(path);
     }
     
     public static void start(String type, long tid) throws KeeperException, InterruptedException {

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooQueueLock.java Mon Jan 14 22:03:24 2013
@@ -16,7 +16,6 @@
  */
 package org.apache.accumulo.server.zookeeper;
 
-import java.nio.charset.Charset;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 
@@ -24,8 +23,6 @@ import org.apache.accumulo.fate.zookeepe
 import org.apache.zookeeper.KeeperException;
 
 public class ZooQueueLock extends org.apache.accumulo.fate.zookeeper.ZooQueueLock {
-
-  private static final Charset utf8 = Charset.forName("UTF8");
   
   public ZooQueueLock(String path, boolean ephemeral) throws KeeperException, InterruptedException {
     super(ZooReaderWriter.getRetryingInstance(), path, ephemeral);
@@ -33,13 +30,13 @@ public class ZooQueueLock extends org.ap
   
   public static void main(String args[]) throws InterruptedException, KeeperException {
     ZooQueueLock lock = new ZooQueueLock("/lock", true);
-    DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes(utf8));
-    DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes(utf8));
-    Lock readLock = rlocker.readLock();
+    DistributedReadWriteLock rlocker = new DistributedReadWriteLock(lock, "reader".getBytes());
+    DistributedReadWriteLock wlocker = new DistributedReadWriteLock(lock, "wlocker".getBytes());
+    final Lock readLock = rlocker.readLock();
     readLock.lock();
-    Lock readLock2 = rlocker.readLock();
+    final Lock readLock2 = rlocker.readLock();
     readLock2.lock();
-    Lock writeLock = wlocker.writeLock();
+    final Lock writeLock = wlocker.writeLock();
     if (writeLock.tryLock(100, TimeUnit.MILLISECONDS))
       throw new RuntimeException("Write lock achieved during read lock!");
     readLock.unlock();
@@ -47,7 +44,7 @@ public class ZooQueueLock extends org.ap
     writeLock.lock();
     if (readLock.tryLock(100, TimeUnit.MILLISECONDS))
       throw new RuntimeException("Read lock achieved during write lock!");
-    Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes(utf8));
+    final Lock writeLock2 = DistributedReadWriteLock.recoverLock(lock, "wlocker".getBytes());
     writeLock2.unlock();
     readLock.lock();
     System.out.println("success");

Modified: accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriter.java Mon Jan 14 22:03:24 2013
@@ -20,7 +20,6 @@ import java.lang.reflect.InvocationHandl
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
-import java.nio.charset.Charset;
 
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
@@ -35,11 +34,9 @@ public class ZooReaderWriter extends org
   private static final String USER = "accumulo";
   private static ZooReaderWriter instance = null;
   private static IZooReaderWriter retryingInstance = null;
-
-  private static final Charset utf8 = Charset.forName("UTF8");
   
   public ZooReaderWriter(String string, int timeInMillis, String secret) {
-    super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(utf8));
+    super(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes());
   }
   
   public static synchronized ZooReaderWriter getInstance() {

Modified: accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java Mon Jan 14 22:03:24 2013
@@ -23,7 +23,7 @@ import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import junit.framework.Assert;
+import org.junit.Assert;
 
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;

Modified: accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java Mon Jan 14 22:03:24 2013
@@ -22,7 +22,7 @@ import java.util.Map.Entry;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import junit.framework.Assert;
+import org.junit.Assert;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.BatchWriter;
@@ -46,7 +46,7 @@ import org.junit.Test;
  */
 public class TestConfirmDeletes {
   
-  AuthInfo auth = new AuthInfo("root", ByteBuffer.wrap("secret".getBytes()), "instance");
+  AuthInfo auth = new AuthInfo("root", ByteBuffer.wrap("".getBytes()), "instance");
 
   SortedSet<String> newSet(String... s) {
     SortedSet<String> result = new TreeSet<String>(Arrays.asList(s));
@@ -99,7 +99,7 @@ public class TestConfirmDeletes {
     load(instance, metadata, deletes);
 
     SimpleGarbageCollector gc = new SimpleGarbageCollector();
-    gc.init(fs, instance, auth);
+    gc.init(fs, instance, auth, false);
     SortedSet<String> candidates = gc.getCandidates();
     Assert.assertEquals(expectedInitial, candidates.size());
     gc.confirmDeletes(candidates);

Modified: accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/logger/LogFileTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/logger/LogFileTest.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/logger/LogFileTest.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/logger/LogFileTest.java Mon Jan 14 22:03:24 2013
@@ -31,6 +31,7 @@ import java.util.Arrays;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.server.data.ServerMutation;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -92,6 +93,15 @@ public class LogFileTest {
     assertEquals(key.seq, 7);
     assertEquals(key.tid, 8);
     assertEquals(value.mutations, Arrays.asList(m));
+    m = new ServerMutation(new Text("row"));
+    m.put(new Text("cf"), new Text("cq"), new ColumnVisibility("vis"), 12345, new Value("value".getBytes()));
+    m.put(new Text("cf"), new Text("cq"), new ColumnVisibility("vis2"), new Value("value".getBytes()));
+    m.putDelete(new Text("cf"), new Text("cq"), new ColumnVisibility("vis2"));
+    readWrite(MUTATION, 8, 9, null, null, new Mutation[] {m}, key, value);
+    assertEquals(key.event, MUTATION);
+    assertEquals(key.seq, 8);
+    assertEquals(key.tid, 9);
+    assertEquals(value.mutations, Arrays.asList(m));
     readWrite(MANY_MUTATIONS, 9, 10, null, null, new Mutation[] {m, m}, key, value);
     assertEquals(key.event, MANY_MUTATIONS);
     assertEquals(key.seq, 9);

Modified: accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java Mon Jan 14 22:03:24 2013
@@ -88,7 +88,7 @@ public class TestMergeState {
   @Test
   public void test() throws Exception {
     Instance instance = new MockInstance();
-    Connector connector = instance.getConnector("root", "secret");
+    Connector connector = instance.getConnector("root", "");
     BatchWriter bw = connector.createBatchWriter("!METADATA", new BatchWriterConfig());
     
     // Create a fake METADATA table with these splits
@@ -112,7 +112,7 @@ public class TestMergeState {
     
     // Read out the TabletLocationStates
     MockCurrentState state = new MockCurrentState(new MergeInfo(new KeyExtent(tableId, new Text("p"), new Text("e")), MergeInfo.Operation.MERGE));
-    AuthInfo auths = new AuthInfo("root", ByteBuffer.wrap("secret".getBytes()), "instance");
+    AuthInfo auths = new AuthInfo("root", ByteBuffer.wrap("".getBytes()), "instance");
     
     // Verify the tablet state: hosted, and count
     MetaDataStateStore metaDataStateStore = new MetaDataStateStore(instance, auths, state);

Modified: accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java Mon Jan 14 22:03:24 2013
@@ -16,10 +16,7 @@
  */
 package org.apache.accumulo.server.master.balancer;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
@@ -32,8 +29,6 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.Map.Entry;
 
-import junit.framework.Assert;
-
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
@@ -198,7 +193,7 @@ public class DefaultLoadBalancerTest {
         servers.get(migration.newServer).extents.add(migration.tablet);
       }
     }
-    Assert.assertEquals(8, moved);
+    assertEquals(8, moved);
   }
   
   @Test
@@ -243,7 +238,7 @@ public class DefaultLoadBalancerTest {
       }
     }
     // average is 58, with 2 at 59: we need 48 more moved to the short server
-    Assert.assertEquals(48, moved);
+    assertEquals(48, moved);
   }
   
   private void checkBalance(List<KeyExtent> metadataTable, Map<TServerInstance,FakeTServer> servers, Map<String,Integer> expectedCounts) {

Modified: accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java (original)
+++ accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/TableLoadBalancerTest.java Mon Jan 14 22:03:24 2013
@@ -25,7 +25,7 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import junit.framework.Assert;
+import org.junit.Assert;
 
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.admin.TableOperations;

Propchange: accumulo/branches/ACCUMULO-259/src/
------------------------------------------------------------------------------
  Merged /accumulo/trunk/src:r1404663-1432174,1432271-1433134
  Merged /accumulo/branches/1.4/src:r1407301,1423032,1423629,1427864
  Merged /accumulo/branches/1.4/src/src:r1407157,1423032,1423624,1427919,1428054
  Merged /accumulo/branches/1.4:r1423032

Propchange: accumulo/branches/ACCUMULO-259/start/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Mon Jan 14 22:03:24 2013
@@ -1,7 +1,5 @@
 target
-
 .classpath
-
 .project
-
 .settings
+accumulo-start.iml

Modified: accumulo/branches/ACCUMULO-259/start/pom.xml
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/start/pom.xml?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/start/pom.xml (original)
+++ accumulo/branches/ACCUMULO-259/start/pom.xml Mon Jan 14 22:03:24 2013
@@ -46,10 +46,65 @@
             </archive>
           </configuration>
         </plugin>
+		<plugin>
+			<artifactId>maven-surefire-plugin</artifactId>
+			<configuration>
+				<forkMode>always</forkMode>
+			</configuration>
+		</plugin>
+        
       </plugins>
     </pluginManagement>
   </build>
-
+  
+  <profiles>
+    <!-- profile for building against Hadoop 1.0.x
+    Activate by not specifying hadoop.profile -->
+    <profile>
+      <id>hadoop-1.0</id>
+      <activation>
+        <property>
+          <name>!hadoop.profile</name>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-core</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-test</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+    <!-- profile for building against Hadoop 2.0.x
+    Activate using: mvn -Dhadoop.profile=2.0 -->
+    <profile>
+      <id>hadoop-2.0</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>2.0</value>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-client</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+  
   <dependencies>
     <dependency>
       <groupId>log4j</groupId>
@@ -60,13 +115,50 @@
       <artifactId>commons-logging</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-jci-fam</artifactId>
-    </dependency>
-    <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
     </dependency>
+	<dependency>
+		<groupId>org.apache.commons</groupId>
+		<artifactId>commons-vfs2</artifactId>
+		<version>2.0</version>
+	</dependency>
+	<!-- TEST DEPENDENCIES -->
+	<dependency>
+		<groupId>junit</groupId>
+		<artifactId>junit</artifactId>
+		<scope>test</scope>
+	</dependency>
+	<dependency>
+		<groupId>javax.ws.rs</groupId>
+		<artifactId>jsr311-api</artifactId>
+		<version>1.0</version>
+		<scope>test</scope>
+	</dependency>
+	<dependency>
+		<groupId>org.slf4j</groupId>
+		<artifactId>slf4j-nop</artifactId>
+		<version>1.7.2</version>
+		<scope>test</scope>
+	</dependency>
+	<dependency>
+		<groupId>org.powermock</groupId>
+		<artifactId>powermock-module-junit4</artifactId>
+		<version>1.4.12</version>
+		<scope>test</scope>
+	</dependency>
+	<dependency>
+		<groupId>org.powermock</groupId>
+		<artifactId>powermock-api-easymock</artifactId>
+		<version>1.4.12</version>
+		<scope>test</scope>
+	</dependency>
+	<dependency>
+		<groupId>org.easymock</groupId>
+		<artifactId>easymock</artifactId>
+		<version>3.1</version>
+		<scope>test</scope>
+	</dependency>    
   </dependencies>
 
 </project>

Modified: accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/Main.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/Main.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/Main.java (original)
+++ accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/Main.java Mon Jan 14 22:03:24 2013
@@ -34,38 +34,44 @@ public class Main {
       final String argsToPass[] = new String[args.length - 1];
       System.arraycopy(args, 1, argsToPass, 0, args.length - 1);
       
-      Class<?> runTMP = null;
-      
       Thread.currentThread().setContextClassLoader(AccumuloClassLoader.getClassLoader());
+
+      Class<?> vfsClassLoader = AccumuloClassLoader.getClassLoader().loadClass("org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader");
+
+      ClassLoader cl = (ClassLoader) vfsClassLoader.getMethod("getClassLoader", new Class[] {}).invoke(null, new Object[] {});
+      
+      Class<?> runTMP = null;
+
+      Thread.currentThread().setContextClassLoader(cl);
       
       if (args[0].equals("master")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.master.Master");
+        runTMP = cl.loadClass("org.apache.accumulo.server.master.Master");
       } else if (args[0].equals("tserver")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.tabletserver.TabletServer");
+        runTMP = cl.loadClass("org.apache.accumulo.server.tabletserver.TabletServer");
       } else if (args[0].equals("shell")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.util.shell.Shell");
+        runTMP = cl.loadClass("org.apache.accumulo.core.util.shell.Shell");
       } else if (args[0].equals("init")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.util.Initialize");
+        runTMP = cl.loadClass("org.apache.accumulo.server.util.Initialize");
       } else if (args[0].equals("admin")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.util.Admin");
+        runTMP = cl.loadClass("org.apache.accumulo.server.util.Admin");
       } else if (args[0].equals("gc")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.gc.SimpleGarbageCollector");
+        runTMP = cl.loadClass("org.apache.accumulo.server.gc.SimpleGarbageCollector");
       } else if (args[0].equals("monitor")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.monitor.Monitor");
+        runTMP = cl.loadClass("org.apache.accumulo.server.monitor.Monitor");
       } else if (args[0].equals("tracer")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.trace.TraceServer");
+        runTMP = cl.loadClass("org.apache.accumulo.server.trace.TraceServer");
       } else if (args[0].equals("classpath")) {
-        AccumuloClassLoader.printClassPath();
+        vfsClassLoader.getMethod("printClassPath", new Class[] {}).invoke(vfsClassLoader, new Object[] {});
         return;
       } else if (args[0].equals("version")) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.Constants");
+        runTMP = cl.loadClass("org.apache.accumulo.core.Constants");
         System.out.println(runTMP.getField("VERSION").get(null));
         return;
       } else if (args[0].equals("rfile-info") ) {
-        runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.file.rfile.PrintInfo");
+        runTMP = cl.loadClass("org.apache.accumulo.core.file.rfile.PrintInfo");
       } else {
         try {
-          runTMP = AccumuloClassLoader.loadClass(args[0]);
+          runTMP = cl.loadClass(args[0]);
         } catch (ClassNotFoundException cnfe) {
           System.out.println("Classname " + args[0] + " not found.  Please make sure you use the wholly qualified package name.");
           System.exit(1);
@@ -96,7 +102,7 @@ public class Main {
       };
       
       Thread t = new Thread(r, args[0]);
-      t.setContextClassLoader(AccumuloClassLoader.getClassLoader());
+      t.setContextClassLoader(cl);
       t.start();
     } catch (Throwable t) {
       System.err.println("Uncaught exception: " + t.getMessage());

Modified: accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/TestMain.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/TestMain.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/TestMain.java (original)
+++ accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/TestMain.java Mon Jan 14 22:03:24 2013
@@ -16,7 +16,7 @@
  */
 package org.apache.accumulo.start;
 
-/** This program tests that the proper exit code is propogated to the shell.
+/** This program tests that the proper exit code is propagated to the shell.
  *
  * $ ./bin/accumulo org.apache.accumulo.start.TestMain
  * $ ./bin/accumulo org.apache.accumulo.start.TestMain badExit

Modified: accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java (original)
+++ accumulo/branches/ACCUMULO-259/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java Mon Jan 14 22:03:24 2013
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -24,20 +24,14 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
-import org.apache.commons.jci.listeners.AbstractFilesystemAlterationListener;
-import org.apache.commons.jci.monitor.FilesystemAlterationObserver;
 import org.apache.log4j.Logger;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -45,188 +39,122 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
 /**
- * This class builds a hierarchy of Classloaders in the form of:
- * 
- * <pre>
- * SystemClassLoader
- *       ^
- *       |
- * URLClassLoader that references the URLs for HADOOP_HOME, ZOOKEEPER_HOME, ACCUMULO_HOME and their associated directories
- *       ^
- *       |
- * URLClassLoader that references ACCUMULO_HOME/lib/ext, locations from the ACCUMULO property general.dynamic.classpaths and $ACCUMULO_XTRAJARS
- * </pre>
- * 
- * The last URLClassLoader in the chain monitors the directories for changes every 3 seconds (default value). If a change occurs, the last URLClassLoader is
- * abandoned and a new one is created. Objects that still reference the abandoned URLClassLoader will cause it to not be garbage collected until those objects
- * are garbage collected. If reload happens often enough it may cause excessive memory usage.
  * 
  */
 public class AccumuloClassLoader {
   
-  private static class Listener extends AbstractFilesystemAlterationListener {
-    
-    private volatile boolean firstCall = true;
-    
-    @Override
-    public void onStop(FilesystemAlterationObserver pObserver) {
-      
-      super.onStop(pObserver);
-      
-      if (firstCall) {
-        // the first time this is called it reports everything that exist as created
-        // so there is no need to do anything
-        synchronized (this) {
-          firstCall = false;
-          this.notifyAll();
-        }
-        return;
-      }
-      
-      if (super.getChangedFiles().size() > 0 || super.getCreatedFiles().size() > 0 || super.getDeletedFiles().size() > 0
-          || super.getChangedDirectories().size() > 0 || super.getCreatedDirectories().size() > 0 || super.getDeletedDirectories().size() > 0) {
-        log.debug("Files have changed, setting loader to null ");
-        loader = null;
-      }
-      
-    }
-    
-    public synchronized void waitForFirstCall() {
-      while (firstCall == true) {
-        try {
-          this.wait();
-        } catch (InterruptedException e) {}
-      }
-    }
-  }
-  
-  private static final Logger log = Logger.getLogger(AccumuloClassLoader.class);
-  
   public static final String CLASSPATH_PROPERTY_NAME = "general.classpaths";
+
+  public static final String ACCUMULO_CLASSPATH_VALUE = "$ACCUMULO_HOME/conf,\n" 
+      + "$ACCUMULO_HOME/lib/[^.].*.jar,\n" 
+      + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" 
+      + "$HADOOP_PREFIX/[^.].*.jar,\n" + "$HADOOP_CONF_DIR,\n"
+      + "$HADOOP_PREFIX/lib/[^.].*.jar,\n";
   
-  public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
-  
-  public static final String ACCUMULO_CLASSPATH_VALUE = "$ACCUMULO_HOME/conf,\n" + "$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,\n"
-      + "$ACCUMULO_HOME/lib/[^.].*.jar,\n" + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" + "$HADOOP_HOME/[^.].*.jar,\n" + "$HADOOP_HOME/conf,\n"
-      + "$HADOOP_HOME/lib/[^.].*.jar,\n";
-  
-  /**
-   * Dynamic classpath. These locations will be monitored for changes.
-   */
-  public static final String DEFAULT_DYNAMIC_CLASSPATH_VALUE = "$ACCUMULO_HOME/lib/ext/[^.].*.jar\n";
-  
-  public static final String DEFAULT_CLASSPATH_VALUE = ACCUMULO_CLASSPATH_VALUE;
+  private static String SITE_CONF;
   
-  private static final String SITE_CONF;
+  private static URLClassLoader classloader;
+
+  private static Logger log = Logger.getLogger(AccumuloClassLoader.class);
+
   static {
     String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
     if (System.getenv("ACCUMULO_HOME") != null) {
       // accumulo home should be set
       SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile;
     } else {
-      /*
-       * if not it must be the build-server in which case I use a hack to get unittests working
-       */
-      String userDir = System.getProperty("user.dir");
-      if (userDir == null)
-        throw new RuntimeException("Property user.dir is not set");
-      int index = userDir.indexOf("accumulo/");
-      if (index >= 0) {
-        String acuhome = userDir.substring(0, index + "accumulo/".length());
-        SITE_CONF = acuhome + "/conf/" + configFile;
-      } else {
-        SITE_CONF = "/conf/" + configFile;
-      }
+      SITE_CONF = null;
     }
-    
+
+    //Register the shutdown hook
+    // TODO
+    // Runtime.getRuntime().addShutdownHook(new Thread(new AccumuloVFSClassLoaderShutdownThread()));
   }
-  
-  private static ClassLoader parent = null;
-  private static volatile ClassLoader loader = null;
-  private static AccumuloFilesystemAlterationMonitor monitor = null;
-  private static Object lock = new Object();
-  
-  private static ArrayList<URL> findDynamicURLs() throws IOException {
-    StringBuilder cp = new StringBuilder(getAccumuloDynamicClasspathStrings());
-    String envJars = System.getenv("ACCUMULO_XTRAJARS");
-    if (null != envJars && !envJars.equals(""))
-      cp = cp.append(",").append(envJars);
-    String[] cps = replaceEnvVars(cp.toString(), System.getenv()).split(",");
-    ArrayList<URL> urls = new ArrayList<URL>();
-    for (String classpath : cps) {
-      if (!classpath.startsWith("#")) {
-        addUrl(classpath, urls);
+
+  /**
+   * Parses and XML Document for a property node for a <name> with the value propertyName if it finds one the function return that property's value for its
+   * <value> node. If not found the function will return null
+   * 
+   * @param d
+   *          XMLDocument to search through
+   * @param propertyName
+   */
+  private static String getAccumuloClassPathStrings(Document d, String propertyName) {
+    NodeList pnodes = d.getElementsByTagName("property");
+    for (int i = pnodes.getLength() - 1; i >= 0; i--) {
+      Element current_property = (Element) pnodes.item(i);
+      Node cname = current_property.getElementsByTagName("name").item(0);
+      if (cname != null && cname.getTextContent().compareTo(propertyName) == 0) {
+        Node cvalue = current_property.getElementsByTagName("value").item(0);
+        if (cvalue != null) {
+          return cvalue.getTextContent();
+        }
       }
     }
-    return urls;
+    return null;
   }
+
+  /**
+   * Looks for the site configuration file for Accumulo and if it has a property for propertyName return it otherwise returns defaultValue Should throw an
+   * exception if the default configuration can not be read;
+   * 
+   * @param propertyName
+   *          Name of the property to pull
+   * @param defaultValue
+   *          Value to default to if not found.
+   * @return site or default class path String
+   */
   
-  private static Set<File> findDirsFromUrls() throws IOException {
-    Set<File> dirs = new HashSet<File>();
-    StringBuilder cp = new StringBuilder(getAccumuloDynamicClasspathStrings());
-    String envJars = System.getenv("ACCUMULO_XTRAJARS");
-    if (null != envJars && !envJars.equals(""))
-      cp = cp.append(",").append(envJars);
-    String[] cps = replaceEnvVars(cp.toString(), System.getenv()).split(",");
-    ArrayList<URL> urls = new ArrayList<URL>();
-    for (String classpath : cps) {
-      if (!classpath.startsWith("#")) {
-        classpath = classpath.trim();
-        if (classpath.length() == 0)
-          continue;
-        
-        classpath = replaceEnvVars(classpath, System.getenv());
-        
-        // Try to make a URI out of the classpath
-        URI uri = null;
-        try {
-          uri = new URI(classpath);
-        } catch (URISyntaxException e) {
-          // Not a valid URI
-        }
-        
-        if (null == uri || !uri.isAbsolute() || (null != uri.getScheme() && uri.getScheme().equals("file://"))) {
-          // Then treat this URI as a File.
-          // This checks to see if the url string is a dir if it expand and get all jars in that directory
-          final File extDir = new File(classpath);
-          if (extDir.isDirectory())
-            urls.add(extDir.toURI().toURL());
-          else {
-            urls.add(extDir.getAbsoluteFile().getParentFile().toURI().toURL());
-          }
-        } else {
-          urls.add(uri.toURL());
-        }
-      }
-    }
+  public static String getAccumuloString(String propertyName, String defaultValue) {
     
-    for (URL url : urls) {
+    try {
+      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilder db = dbf.newDocumentBuilder();
+      String site_classpath_string = null;
       try {
-        File f = new File(url.toURI());
-        if (!f.isDirectory())
-          f = f.getParentFile();
-        dirs.add(f);
-      } catch (URISyntaxException e) {
-        log.error("Unable to find directory for " + url + ", cannot create URI from it");
+        Document site_conf = db.parse(SITE_CONF);
+        site_classpath_string = getAccumuloClassPathStrings(site_conf, propertyName);
+      } catch (Exception e) {
+        /* we don't care because this is optional and we can use defaults */
       }
+      if (site_classpath_string != null)
+        return site_classpath_string;
+      return defaultValue;
+    } catch (Exception e) {
+      throw new IllegalStateException("ClassPath Strings Lookup failed", e);
     }
-    return dirs;
   }
   
-  private static ArrayList<URL> findAccumuloURLs() throws IOException {
-    String cp = getAccumuloClasspathStrings();
-    if (cp == null)
-      return new ArrayList<URL>();
-    String[] cps = replaceEnvVars(cp, System.getenv()).split(",");
-    ArrayList<URL> urls = new ArrayList<URL>();
-    for (String classpath : cps) {
-      if (!classpath.startsWith("#")) {
-        addUrl(classpath, urls);
+  /**
+   * Replace environment variables in the classpath string with their actual value
+   * 
+   * @param classpath
+   * @param env
+   * @return
+   */
+  public static String replaceEnvVars(String classpath, Map<String,String> env) {
+    Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*");
+    Matcher envMatcher = envPat.matcher(classpath);
+    while (envMatcher.find(0)) {
+      // name comes after the '$'
+      String varName = envMatcher.group().substring(1);
+      String varValue = env.get(varName);
+      if (varValue == null) {
+        varValue = "";
       }
+      classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end()));
+      envMatcher.reset(classpath);
     }
-    return urls;
+    return classpath;
   }
-  
+
+  /**
+   * Populate the list of URLs with the items in the classpath string
+   * 
+   * @param classpath
+   * @param urls
+   * @throws MalformedURLException
+   */
   private static void addUrl(String classpath, ArrayList<URL> urls) throws MalformedURLException {
     classpath = classpath.trim();
     if (classpath.length() == 0)
@@ -258,7 +186,11 @@ public class AccumuloClassLoader {
           if (extJars != null && extJars.length > 0) {
             for (File jar : extJars)
               urls.add(jar.toURI().toURL());
+          } else {
+            log.warn("ignoring classpath entry " + classpath);
           }
+        } else {
+          log.warn("ignoring classpath entry " + classpath);
         }
       }
     } else {
@@ -266,170 +198,48 @@ public class AccumuloClassLoader {
     }
     
   }
-  
-  private static String replaceEnvVars(String classpath, Map<String,String> env) {
-    Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*");
-    Matcher envMatcher = envPat.matcher(classpath);
-    while (envMatcher.find(0)) {
-      // name comes after the '$'
-      String varName = envMatcher.group().substring(1);
-      String varValue = env.get(varName);
-      if (varValue == null) {
-        varValue = "";
-      }
-      classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end()));
-      envMatcher.reset(classpath);
-    }
-    return classpath;
-  }
-  
-  public static String getAccumuloDynamicClasspathStrings() throws IllegalStateException {
-    return getAccumuloString(DYNAMIC_CLASSPATH_PROPERTY_NAME, DEFAULT_DYNAMIC_CLASSPATH_VALUE);
-  }
-  
-  public static String getAccumuloClasspathStrings() throws IllegalStateException {
-    return getAccumuloString(CLASSPATH_PROPERTY_NAME, ACCUMULO_CLASSPATH_VALUE);
-  }
-  
-  /**
-   * Looks for the site configuration file for Accumulo and if it has a property for propertyName return it otherwise returns defaultValue Should throw an
-   * exception if the default configuration can not be read;
-   * 
-   * @param propertyName
-   *          Name of the property to pull
-   * @param defaultValue
-   *          Value to default to if not found.
-   * @return site or default class path String
-   */
-  
-  private static String getAccumuloString(String propertyName, String defaultValue) {
-    try {
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-      DocumentBuilder db = dbf.newDocumentBuilder();
-      String site_classpath_string = null;
-      try {
-        Document site_conf = db.parse(SITE_CONF);
-        site_classpath_string = getAccumuloClassPathStrings(site_conf, propertyName);
-      } catch (Exception e) {
-        /* we don't care because this is optional and we can use defaults */
-      }
-      if (site_classpath_string != null)
-        return site_classpath_string;
-      return defaultValue;
-    } catch (Exception e) {
-      throw new IllegalStateException("ClassPath Strings Lookup failed", e);
-    }
-  }
-  
-  /**
-   * Parses and XML Document for a property node for a <name> with the value propertyName if it finds one the function return that property's value for its
-   * <value> node. If not found the function will return null
-   * 
-   * @param d
-   *          XMLDocument to search through
-   * @param propertyName
-   */
-  private static String getAccumuloClassPathStrings(Document d, String propertyName) {
-    NodeList pnodes = d.getElementsByTagName("property");
-    for (int i = pnodes.getLength() - 1; i >= 0; i--) {
-      Element current_property = (Element) pnodes.item(i);
-      Node cname = current_property.getElementsByTagName("name").item(0);
-      if (cname != null && cname.getTextContent().compareTo(propertyName) == 0) {
-        Node cvalue = current_property.getElementsByTagName("value").item(0);
-        if (cvalue != null) {
-          return cvalue.getTextContent();
-        }
-      }
-    }
-    return null;
-  }
-  
-  public static void printClassPath() {
-    try {
-      System.out.println("Accumulo List of classpath items are:");
-      for (URL url : findDynamicURLs()) {
-        System.out.println(url.toExternalForm());
-      }
-      for (URL url : findAccumuloURLs()) {
-        System.out.println(url.toExternalForm());
+    
+  private static ArrayList<URL> findAccumuloURLs() throws IOException {
+    String cp = getAccumuloString(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE);
+    if (cp == null)
+      return new ArrayList<URL>();
+    String[] cps = replaceEnvVars(cp, System.getenv()).split(",");
+    ArrayList<URL> urls = new ArrayList<URL>();
+    for (String classpath : cps) {
+      if (!classpath.startsWith("#")) {
+        addUrl(classpath, urls);
       }
-    } catch (Throwable t) {
-      throw new RuntimeException(t);
     }
+    return urls;
   }
   
-  public synchronized static <U> Class<? extends U> loadClass(String classname, Class<U> extension) throws ClassNotFoundException {
-    try {
-      return (Class<? extends U>) Class.forName(classname, true, getClassLoader()).asSubclass(extension);
-    } catch (IOException e) {
-      throw new ClassNotFoundException("IO Error loading class " + classname, e);
-    }
-  }
-  
-  public static Class<?> loadClass(String classname) throws ClassNotFoundException {
-    return loadClass(classname, Object.class).asSubclass(Object.class);
-  }
-  
-  private static ClassLoader getAccumuloClassLoader() throws IOException {
-    ClassLoader parentClassLoader = ClassLoader.getSystemClassLoader();
-    ArrayList<URL> accumuloURLs = findAccumuloURLs();
-    log.debug("Create Dependency ClassLoader using URLs: " + accumuloURLs.toString());
-    URLClassLoader aClassLoader = new URLClassLoader(accumuloURLs.toArray(new URL[accumuloURLs.size()]), parentClassLoader);
-    return aClassLoader;
-  }
-  
-  public static ClassLoader getClassLoader() throws IOException {
-    ClassLoader localLoader = loader;
-    while (null == localLoader) {
-      synchronized (lock) {
-        if (null == loader) {
-          
-          if (null != monitor) {
-            monitor.stop();
-            monitor = null;
-          }
-          
-          if (null == parent)
-            parent = getAccumuloClassLoader();
-          
-          // Find the dynamic classpath items
-          final ArrayList<URL> dynamicURLs = findDynamicURLs();
-          // Find the directories for the dynamic classpath items
-          Set<File> monitoredDirs = findDirsFromUrls();
-          
-          // Setup the directory monitor
-          monitor = new AccumuloFilesystemAlterationMonitor();
-          Listener myListener = new Listener();
-          for (File dir : monitoredDirs) {
-            if (monitor.getListenersFor(dir) != null || monitor.getListenersFor(dir).length > 0) {
-              log.debug("Monitor listening to " + dir.getAbsolutePath());
-              monitor.addListener(dir, myListener);
-            }
-          }
-          // setting this interval below 1 second is probably not helpful because the file modification time is in seconds
-          monitor.setInterval(1000);
-          monitor.start();
-          myListener.waitForFirstCall();
-          log.debug("Create Dynamic ClassLoader using URLs: " + dynamicURLs.toString());
-          // the file system listner must run once to get the state of the filesystem, and it only detects changes on subsequent runs
-          // we need to check to see if the set of files has changed since we started monitoring, so that we know we are monitoring all of the files
-          // in the
-          // class loader
-          HashSet<URL> checkDynamicURLs = new HashSet<URL>(findDynamicURLs());
-          HashSet<URL> originalDynamicURLs = new HashSet<URL>(dynamicURLs);
-          if (checkDynamicURLs.equals(originalDynamicURLs)) {
-            // file set has not changed, so we know that all of the dynamicURLs are being monitored for changes
-            loader = AccessController.doPrivileged(new PrivilegedAction<ClassLoader>() {
-              public ClassLoader run() {
-                return new URLClassLoader(dynamicURLs.toArray(new URL[dynamicURLs.size()]), parent);
+  public static synchronized ClassLoader getClassLoader() throws IOException {
+    if (classloader == null) {
+      ArrayList<URL> urls = findAccumuloURLs();
+      
+      ClassLoader parentClassLoader = ClassLoader.getSystemClassLoader();
+
+      log.debug("Create 2nd tier ClassLoader using URLs: " + urls.toString());
+      URLClassLoader aClassLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader) {
+        protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+
+          if (name.startsWith("org.apache.accumulo.start.classloader.vfs")) {
+            Class<?> c = findLoadedClass(name);
+            if (c == null) {
+              try {
+                // try finding this class here instead of parent
+                c = findClass(name);
+              } catch (ClassNotFoundException e) {
+                
               }
-            });
+            }
           }
+          return super.loadClass(name, resolve);
         }
-      }
-      localLoader = loader;
+      };
+      classloader = aClassLoader;
     }
-    return localLoader;
+    
+    return classloader;
   }
-  
 }

Propchange: accumulo/branches/ACCUMULO-259/test/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jan 14 22:03:24 2013
@@ -0,0 +1,5 @@
+target
+.classpath
+.project
+.settings
+accumulo-test.iml

Copied: accumulo/branches/ACCUMULO-259/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java (from r1432174, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java)
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java?p2=accumulo/branches/ACCUMULO-259/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java&p1=accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java&r1=1432174&r2=1433166&rev=1433166&view=diff
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java (original)
+++ accumulo/branches/ACCUMULO-259/test/src/main/java/org/apache/accumulo/test/MiniAccumuloCluster.java Mon Jan 14 22:03:24 2013
@@ -26,6 +26,7 @@ import java.io.InputStreamReader;
 import java.net.ServerSocket;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -102,7 +103,6 @@ public class MiniAccumuloCluster {
     }
   }
 
-  private File baseDir;
   private File libDir;
   private File confDir;
   private File zooKeeperDir;
@@ -113,15 +113,13 @@ public class MiniAccumuloCluster {
   
   private Process zooKeeperProcess;
   private Process masterProcess;
-  private Process tabletServerProcess;
-  private Process loggerProcess;
 
   private int zooKeeperPort;
   
   private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
-  private String rootPassword;
-
 
+  private MacConfig config;
+  private Process[] tabletServerProcesses;
 
   private int getRandomFreePort() {
     Random r = new Random();
@@ -166,7 +164,7 @@ public class MiniAccumuloCluster {
     
     ProcessBuilder builder = new ProcessBuilder(argList);
     
-    builder.environment().put("ACCUMULO_HOME", baseDir.getAbsolutePath());
+    builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
     builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
 
     Process process = builder.start();
@@ -204,23 +202,22 @@ public class MiniAccumuloCluster {
    * @throws IOException
    */
 
-  public MiniAccumuloCluster(File dir, String rootPassword, Map<String,String> siteConfig) throws IOException {
+  public MiniAccumuloCluster(MacConfig config) throws IOException {
 
-    if (dir.exists() && !dir.isDirectory())
-      throw new IllegalArgumentException("Must pass in directory, " + dir + " is a file");
+    if (config.getDir().exists() && !config.getDir().isDirectory())
+      throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
     
-    if (dir.exists() && dir.list().length != 0)
-      throw new IllegalArgumentException("Directory " + dir + " is not empty");
+    if (config.getDir().exists() && config.getDir().list().length != 0)
+      throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
     
-    this.rootPassword = rootPassword;
-
-    baseDir = dir;
-    libDir = new File(dir, "lib");
-    confDir = new File(dir, "conf");
-    accumuloDir = new File(dir, "accumulo");
-    zooKeeperDir = new File(dir, "zookeeper");
-    logDir = new File(dir, "logs");
-    walogDir = new File(dir, "walogs");
+    this.config = config;
+
+    libDir = new File(config.getDir(), "lib");
+    confDir = new File(config.getDir(), "conf");
+    accumuloDir = new File(config.getDir(), "accumulo");
+    zooKeeperDir = new File(config.getDir(), "zookeeper");
+    logDir = new File(config.getDir(), "logs");
+    walogDir = new File(config.getDir(), "walogs");
     
     confDir.mkdirs();
     accumuloDir.mkdirs();
@@ -236,12 +233,15 @@ public class MiniAccumuloCluster {
     FileWriter fileWriter = new FileWriter(siteFile);
     fileWriter.append("<configuration>\n");
     
+    HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
     appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
     appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
     appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
     appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
     appendProp(fileWriter, Property.MASTER_CLIENTPORT, "" + getRandomFreePort(), siteConfig);
     appendProp(fileWriter, Property.TSERV_CLIENTPORT, "" + getRandomFreePort(), siteConfig);
+    appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
     appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
     appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
     appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
@@ -305,13 +305,17 @@ public class MiniAccumuloCluster {
     // sleep a little bit to let zookeeper come up before calling init, seems to work better
     UtilWaitThread.sleep(250);
 
-    Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", rootPassword);
+    Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", config.getRootPassword());
     int ret = initProcess.waitFor();
     if (ret != 0) {
       throw new RuntimeException("Initialize process returned " + ret);
     }
-   
-    tabletServerProcess = exec(TabletServer.class);
+    
+    tabletServerProcesses = new Process[config.getNumTservers()];
+    for (int i = 0; i < config.getNumTservers(); i++) {
+      tabletServerProcesses[i] = exec(TabletServer.class);
+    }
+
     masterProcess = exec(Master.class);
   }
 
@@ -344,10 +348,11 @@ public class MiniAccumuloCluster {
       zooKeeperProcess.destroy();
     if (masterProcess != null)
       masterProcess.destroy();
-    if (tabletServerProcess != null)
-      tabletServerProcess.destroy();
-    if (loggerProcess != null)
-      loggerProcess.destroy();
+    if (tabletServerProcesses != null) {
+      for (Process tserver : tabletServerProcesses) {
+        tserver.destroy();
+      }
+    }
     
     for (LogWriter lw : logWriters)
       lw.flush();

Copied: accumulo/branches/ACCUMULO-259/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java (from r1432174, accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java)
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java?p2=accumulo/branches/ACCUMULO-259/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java&p1=accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java&r1=1432174&r2=1433166&rev=1433166&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java (original)
+++ accumulo/branches/ACCUMULO-259/test/src/test/java/org/apache/accumulo/test/MiniAccumuloClusterTest.java Mon Jan 14 22:03:24 2013
@@ -18,7 +18,6 @@ package org.apache.accumulo.test;
 
 import java.io.File;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map.Entry;
 import java.util.UUID;
 
@@ -60,7 +59,7 @@ public class MiniAccumuloClusterTest {
     
     Logger.getLogger("org.apache.zookeeper").setLevel(Level.WARN);
 
-    accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret", new HashMap<String,String>());
+    accumulo = new MiniAccumuloCluster(new MacConfig(folder.getRoot(), "superSecret").setNumTservers(2));
     
     accumulo.start();
     

Modified: accumulo/branches/ACCUMULO-259/test/system/auto/JavaTest.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/test/system/auto/JavaTest.py?rev=1433166&r1=1433165&r2=1433166&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/test/system/auto/JavaTest.py (original)
+++ accumulo/branches/ACCUMULO-259/test/system/auto/JavaTest.py Mon Jan 14 22:03:24 2013
@@ -49,7 +49,7 @@ class JavaTest(TestUtilsMixin, unittest.
         assert handle.returncode==0
 
     def runJTest(self,host, cmd):
-        return self.runClassOn(host, 'org.apache.accumulo.server.test.functional.FunctionalTest', ['-m',host,'-u',ROOT,'-p',ROOT_PASSWORD,'-i',INSTANCE_NAME,self.testClass,cmd])
+        return self.runClassOn(host, 'org.apache.accumulo.server.test.functional.FunctionalTest', ['-u',ROOT,'-p',ROOT_PASSWORD,'-i',INSTANCE_NAME,'--classname', self.testClass,'--opt', cmd])
         
     def runTest(self):
         handle = self.runJTest(self.masterHost(),'run')



Mime
View raw message