accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [23/43] accumulo git commit: ACCUMULO-3871 move ITs into distro jar, stop building test jar
Date Thu, 04 Jun 2015 18:53:04 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java b/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
new file mode 100644
index 0000000..88224b5
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
@@ -0,0 +1,731 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.master.replication.UnorderedWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class UnorderedWorkAssignerReplicationIT extends ConfigurableMacBase {
+  private static final Logger log = LoggerFactory.getLogger(UnorderedWorkAssignerReplicationIT.class);
+
+  private ExecutorService executor;
+  private int timeoutFactor = 1;
+
+  @Before
+  public void createExecutor() {
+    executor = Executors.newSingleThreadExecutor();
+
+    try {
+      timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
+    } catch (NumberFormatException exception) {
+      log.warn("Could not parse timeout.factor, not increasing timeout.");
+    }
+
+    Assert.assertTrue("The timeout factor must be a positive, non-zero value", timeoutFactor > 0);
+  }
+
+  @After
+  public void stopExecutor() {
+    if (null != executor) {
+      executor.shutdownNow();
+    }
+  }
+
+  @Override
+  public int defaultTimeoutSeconds() {
+    return 60 * 5;
+  }
+
+  @Override
+  public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+    cfg.setNumTservers(1);
+    cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
+    cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+    cfg.setProperty(Property.GC_CYCLE_START, "1s");
+    cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+    cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+    cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+    cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+    cfg.setProperty(Property.REPLICATION_NAME, "master");
+    cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, UnorderedWorkAssigner.class.getName());
+    cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+    hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+  }
+
+  /**
+   * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+   */
+  private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+    // Set the same SSL information from the primary when present
+    Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+    if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+      Map<String,String> peerSiteConfig = new HashMap<String,String>();
+      peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+      String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+      Assert.assertNotNull("Keystore Path was null", keystorePath);
+      peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+      String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+      Assert.assertNotNull("Truststore Path was null", truststorePath);
+      peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+      // Passwords might be stored in CredentialProvider
+      String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+      if (null != keystorePassword) {
+        peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+      }
+      String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+      if (null != truststorePassword) {
+        peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+      }
+
+      System.out.println("Setting site configuration for peer " + peerSiteConfig);
+      peerCfg.setSiteConfig(peerSiteConfig);
+    }
+
+    // Use the CredentialProvider if the primary also uses one
+    String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+    if (null != credProvider) {
+      Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+      peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+      peerCfg.setSiteConfig(peerSiteConfig);
+    }
+  }
+
+  @Test
+  public void dataWasReplicatedToThePeer() throws Exception {
+    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+        ROOT_PASSWORD);
+    peerCfg.setNumTservers(1);
+    peerCfg.setInstanceName("peer");
+    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+    peerCluster.start();
+
+    try {
+      final Connector connMaster = getConnector();
+      final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+      ReplicationTable.setOnline(connMaster);
+
+      String peerUserName = "peer", peerPassword = "foo";
+
+      String peerClusterName = "peer";
+
+      connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+      connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+      connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+      // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+      connMaster.instanceOperations().setProperty(
+          Property.REPLICATION_PEERS.getKey() + peerClusterName,
+          ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+              AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+      final String masterTable = "master", peerTable = "peer";
+
+      connMaster.tableOperations().create(masterTable);
+      String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+      Assert.assertNotNull(masterTableId);
+
+      connPeer.tableOperations().create(peerTable);
+      String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+      Assert.assertNotNull(peerTableId);
+
+      connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+      // Replicate this table to the peerClusterName in a table with the peerTableId table id
+      connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+      connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+      // Wait for zookeeper updates (configuration) to propagate
+      UtilWaitThread.sleep(3 * 1000);
+
+      // Write some data to table1
+      BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+      for (int rows = 0; rows < 5000; rows++) {
+        Mutation m = new Mutation(Integer.toString(rows));
+        for (int cols = 0; cols < 100; cols++) {
+          String value = Integer.toString(cols);
+          m.put(value, "", value);
+        }
+        bw.addMutation(m);
+      }
+
+      bw.close();
+
+      log.info("Wrote all data to master cluster");
+
+      final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+      for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+        cluster.killProcess(ServerType.TABLET_SERVER, proc);
+      }
+      cluster.exec(TabletServer.class);
+
+      log.info("TabletServer restarted");
+      Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+      log.info("TabletServer is online");
+
+      log.info("");
+      log.info("Fetching metadata records:");
+      for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+        if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+          log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+        } else {
+          log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+        }
+      }
+
+      log.info("");
+      log.info("Fetching replication records:");
+      for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+        log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+      }
+
+      Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+        @Override
+        public Boolean call() throws Exception {
+          connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+          log.info("Drain completed");
+          return true;
+        }
+
+      });
+
+      long timeoutSeconds = timeoutFactor * 30;
+      try {
+        future.get(timeoutSeconds, TimeUnit.SECONDS);
+      } catch (TimeoutException e) {
+        future.cancel(true);
+        Assert.fail("Drain did not finish within " + timeoutSeconds + " seconds");
+      }
+
+      log.info("drain completed");
+
+      log.info("");
+      log.info("Fetching metadata records:");
+      for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+        if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+          log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+        } else {
+          log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+        }
+      }
+
+      log.info("");
+      log.info("Fetching replication records:");
+      for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+        log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+      }
+
+      Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+      Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+      Entry<Key,Value> masterEntry = null, peerEntry = null;
+      while (masterIter.hasNext() && peerIter.hasNext()) {
+        masterEntry = masterIter.next();
+        peerEntry = peerIter.next();
+        Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+            masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+        Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+      }
+
+      log.info("Last master entry: " + masterEntry);
+      log.info("Last peer entry: " + peerEntry);
+
+      Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+      Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+    } finally {
+      peerCluster.stop();
+    }
+  }
+
+  @Test
+  public void dataReplicatedToCorrectTable() throws Exception {
+    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+        ROOT_PASSWORD);
+    peerCfg.setNumTservers(1);
+    peerCfg.setInstanceName("peer");
+    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+    peer1Cluster.start();
+
+    try {
+      Connector connMaster = getConnector();
+      Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+      String peerClusterName = "peer";
+      String peerUserName = "peer", peerPassword = "foo";
+
+      // Create local user
+      connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+      connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+      connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+      // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+      connMaster.instanceOperations().setProperty(
+          Property.REPLICATION_PEERS.getKey() + peerClusterName,
+          ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+              AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+      String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+      // Create tables
+      connMaster.tableOperations().create(masterTable1);
+      String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+      Assert.assertNotNull(masterTableId1);
+
+      connMaster.tableOperations().create(masterTable2);
+      String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+      Assert.assertNotNull(masterTableId2);
+
+      connPeer.tableOperations().create(peerTable1);
+      String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+      Assert.assertNotNull(peerTableId1);
+
+      connPeer.tableOperations().create(peerTable2);
+      String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+      Assert.assertNotNull(peerTableId2);
+
+      // Grant write permission
+      connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+      connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+      // Replicate this table to the peerClusterName in a table with the peerTableId table id
+      connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+      connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+      connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+      connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+      // Wait for zookeeper updates (configuration) to propogate
+      UtilWaitThread.sleep(3 * 1000);
+
+      // Write some data to table1
+      BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+      long masterTable1Records = 0l;
+      for (int rows = 0; rows < 2500; rows++) {
+        Mutation m = new Mutation(masterTable1 + rows);
+        for (int cols = 0; cols < 100; cols++) {
+          String value = Integer.toString(cols);
+          m.put(value, "", value);
+          masterTable1Records++;
+        }
+        bw.addMutation(m);
+      }
+
+      bw.close();
+
+      // Write some data to table2
+      bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+      long masterTable2Records = 0l;
+      for (int rows = 0; rows < 2500; rows++) {
+        Mutation m = new Mutation(masterTable2 + rows);
+        for (int cols = 0; cols < 100; cols++) {
+          String value = Integer.toString(cols);
+          m.put(value, "", value);
+          masterTable2Records++;
+        }
+        bw.addMutation(m);
+      }
+
+      bw.close();
+
+      log.info("Wrote all data to master cluster");
+
+      Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+          masterTable2);
+
+      while (!ReplicationTable.isOnline(connMaster)) {
+        Thread.sleep(500);
+      }
+
+      // Restart the tserver to force a close on the WAL
+      for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+        cluster.killProcess(ServerType.TABLET_SERVER, proc);
+      }
+      cluster.exec(TabletServer.class);
+
+      log.info("Restarted the tserver");
+
+      // Read the data -- the tserver is back up and running
+      Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+      // Wait for both tables to be replicated
+      log.info("Waiting for {} for {}", filesFor1, masterTable1);
+      connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+      log.info("Waiting for {} for {}", filesFor2, masterTable2);
+      connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+      long countTable = 0l;
+      for (int i = 0; i < 5; i++) {
+        countTable = 0l;
+        for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+          countTable++;
+          Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+              .startsWith(masterTable1));
+        }
+
+        log.info("Found {} records in {}", countTable, peerTable1);
+
+        if (masterTable1Records != countTable) {
+          log.warn("Did not find {} expected records in {}, only found {}", masterTable1Records, peerTable1, countTable);
+        }
+      }
+
+      Assert.assertEquals(masterTable1Records, countTable);
+
+      for (int i = 0; i < 5; i++) {
+        countTable = 0l;
+        for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+          countTable++;
+          Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+              .startsWith(masterTable2));
+        }
+
+        log.info("Found {} records in {}", countTable, peerTable2);
+
+        if (masterTable2Records != countTable) {
+          log.warn("Did not find {} expected records in {}, only found {}", masterTable2Records, peerTable2, countTable);
+        }
+      }
+
+      Assert.assertEquals(masterTable2Records, countTable);
+
+    } finally {
+      peer1Cluster.stop();
+    }
+  }
+
+  @Test
+  public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+        ROOT_PASSWORD);
+    peerCfg.setNumTservers(1);
+    peerCfg.setInstanceName("peer");
+    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+    MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+    peerCluster.start();
+
+    Connector connMaster = getConnector();
+    Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+    String peerUserName = "repl";
+    String peerPassword = "passwd";
+
+    // Create a user on the peer for replication to use
+    connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+    String peerClusterName = "peer";
+
+    // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+    connMaster.instanceOperations().setProperty(
+        Property.REPLICATION_PEERS.getKey() + peerClusterName,
+        ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+            AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+    // Configure the credentials we should use to authenticate ourselves to the peer for replication
+    connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+    connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+    String masterTable = "master", peerTable = "peer";
+
+    connMaster.tableOperations().create(masterTable);
+    String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+    Assert.assertNotNull(masterTableId);
+
+    connPeer.tableOperations().create(peerTable);
+    String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+    Assert.assertNotNull(peerTableId);
+
+    // Give our replication user the ability to write to the table
+    connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+    // Replicate this table to the peerClusterName in a table with the peerTableId table id
+    connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+    connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+    // Write some data to table1
+    BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+    for (int rows = 0; rows < 5000; rows++) {
+      Mutation m = new Mutation(Integer.toString(rows));
+      for (int cols = 0; cols < 100; cols++) {
+        String value = Integer.toString(cols);
+        m.put(value, "", value);
+      }
+      bw.addMutation(m);
+    }
+
+    bw.close();
+
+    log.info("Wrote all data to master cluster");
+
+    Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+    for (String s : files) {
+      log.info("Found referenced file for " + masterTable + ": " + s);
+    }
+
+    for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+      cluster.killProcess(ServerType.TABLET_SERVER, proc);
+    }
+
+    cluster.exec(TabletServer.class);
+
+    Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+    for (Entry<Key,Value> kv : connMaster.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
+      log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+    }
+
+    connMaster.replicationOperations().drain(masterTable, files);
+
+    Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+    Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+    Assert.assertTrue("No data in master table", masterIter.hasNext());
+    Assert.assertTrue("No data in peer table", peerIter.hasNext());
+    while (masterIter.hasNext() && peerIter.hasNext()) {
+      Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+      Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+          masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+      Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+    }
+
+    Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+    Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+    peerCluster.stop();
+  }
+
+  @Test
+  public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+    MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+        ROOT_PASSWORD);
+    peerCfg.setNumTservers(1);
+    peerCfg.setInstanceName("peer");
+    updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+    peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+    MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+    peer1Cluster.start();
+
+    try {
+      Connector connMaster = getConnector();
+      Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+      String peerClusterName = "peer";
+
+      String peerUserName = "repl";
+      String peerPassword = "passwd";
+
+      // Create a user on the peer for replication to use
+      connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+      // Configure the credentials we should use to authenticate ourselves to the peer for replication
+      connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+      connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+      // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+      connMaster.instanceOperations().setProperty(
+          Property.REPLICATION_PEERS.getKey() + peerClusterName,
+          ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+              AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+      String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+      connMaster.tableOperations().create(masterTable1);
+      String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+      Assert.assertNotNull(masterTableId1);
+
+      connMaster.tableOperations().create(masterTable2);
+      String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+      Assert.assertNotNull(masterTableId2);
+
+      connPeer.tableOperations().create(peerTable1);
+      String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+      Assert.assertNotNull(peerTableId1);
+
+      connPeer.tableOperations().create(peerTable2);
+      String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+      Assert.assertNotNull(peerTableId2);
+
+      // Give our replication user the ability to write to the tables
+      connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+      connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+      // Replicate this table to the peerClusterName in a table with the peerTableId table id
+      connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+      connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+      connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+      connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+      // Wait for zookeeper updates (configuration) to propagate
+      UtilWaitThread.sleep(3 * 1000);
+
+      // Write some data to table1
+      BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+      for (int rows = 0; rows < 2500; rows++) {
+        Mutation m = new Mutation(masterTable1 + rows);
+        for (int cols = 0; cols < 100; cols++) {
+          String value = Integer.toString(cols);
+          m.put(value, "", value);
+        }
+        bw.addMutation(m);
+      }
+
+      bw.close();
+
+      // Write some data to table2
+      bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+      for (int rows = 0; rows < 2500; rows++) {
+        Mutation m = new Mutation(masterTable2 + rows);
+        for (int cols = 0; cols < 100; cols++) {
+          String value = Integer.toString(cols);
+          m.put(value, "", value);
+        }
+        bw.addMutation(m);
+      }
+
+      bw.close();
+
+      log.info("Wrote all data to master cluster");
+
+      while (!ReplicationTable.isOnline(connMaster)) {
+        Thread.sleep(500);
+      }
+
+      for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+        cluster.killProcess(ServerType.TABLET_SERVER, proc);
+      }
+
+      cluster.exec(TabletServer.class);
+
+      // Wait until we fully replicated something
+      boolean fullyReplicated = false;
+      for (int i = 0; i < 10 && !fullyReplicated; i++) {
+        UtilWaitThread.sleep(timeoutFactor * 2000);
+
+        Scanner s = ReplicationTable.getScanner(connMaster);
+        WorkSection.limit(s);
+        for (Entry<Key,Value> entry : s) {
+          Status status = Status.parseFrom(entry.getValue().get());
+          if (StatusUtil.isFullyReplicated(status)) {
+            fullyReplicated |= true;
+          }
+        }
+      }
+
+      Assert.assertNotEquals(0, fullyReplicated);
+
+      long countTable = 0l;
+
+      // Check a few times
+      for (int i = 0; i < 10; i++) {
+        countTable = 0l;
+        for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+          countTable++;
+          Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+              .startsWith(masterTable1));
+        }
+        log.info("Found {} records in {}", countTable, peerTable1);
+        if (0 < countTable) {
+          break;
+        }
+        Thread.sleep(2000);
+      }
+
+      Assert.assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
+
+      for (int i = 0; i < 10; i++) {
+        countTable = 0l;
+        for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+          countTable++;
+          Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+              .startsWith(masterTable2));
+        }
+
+        log.info("Found {} records in {}", countTable, peerTable2);
+        if (0 < countTable) {
+          break;
+        }
+        Thread.sleep(2000);
+      }
+      Assert.assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
+
+    } finally {
+      peer1Cluster.stop();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java b/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
new file mode 100644
index 0000000..59197de
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.data.ServerMutation;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.logger.LogEvents;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+public class UnusedWalDoesntCloseReplicationStatusIT extends ConfigurableMacBase {
+
+  @Override
+  public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+    cfg.setNumTservers(1);
+  }
+
+  @Test
+  public void test() throws Exception {
+    File accumuloDir = this.getCluster().getConfig().getAccumuloDir();
+    final Connector conn = getConnector();
+    final String tableName = getUniqueNames(1)[0];
+
+    conn.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+    conn.tableOperations().create(tableName);
+
+    final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+    final int numericTableId = Integer.parseInt(tableId);
+    final int fakeTableId = numericTableId + 1;
+
+    Assert.assertNotNull("Did not find table ID", tableId);
+
+    conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION.getKey(), "true");
+    conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+    // just sleep
+    conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
+        ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
+
+    FileSystem fs = FileSystem.getLocal(new Configuration());
+    File tserverWalDir = new File(accumuloDir, ServerConstants.WAL_DIR + Path.SEPARATOR + "faketserver+port");
+    File tserverWal = new File(tserverWalDir, UUID.randomUUID().toString());
+    fs.mkdirs(new Path(tserverWalDir.getAbsolutePath()));
+
+    // Make a fake WAL with no data in it for our real table
+    FSDataOutputStream out = fs.create(new Path(tserverWal.getAbsolutePath()));
+
+    out.write(DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8));
+
+    DataOutputStream dos = new DataOutputStream(out);
+    dos.writeUTF("NullCryptoModule");
+
+    // Fake a single update WAL that has a mutation for another table
+    LogFileKey key = new LogFileKey();
+    LogFileValue value = new LogFileValue();
+
+    key.event = OPEN;
+    key.tserverSession = tserverWal.getAbsolutePath();
+    key.filename = tserverWal.getAbsolutePath();
+    key.write(out);
+    value.write(out);
+
+    key.event = LogEvents.DEFINE_TABLET;
+    key.tablet = new KeyExtent(new Text(Integer.toString(fakeTableId)), null, null);
+    key.seq = 1l;
+    key.tid = 1;
+
+    key.write(dos);
+    value.write(dos);
+
+    key.tablet = null;
+    key.event = LogEvents.MUTATION;
+    key.filename = tserverWal.getAbsolutePath();
+    value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
+
+    key.write(dos);
+    value.write(dos);
+
+    key.event = LogEvents.COMPACTION_START;
+    key.filename = accumuloDir.getAbsolutePath() + "/tables/" + fakeTableId + "/t-000001/A000001.rf";
+    value.mutations = Collections.emptyList();
+
+    key.write(dos);
+    value.write(dos);
+
+    key.event = LogEvents.COMPACTION_FINISH;
+    value.mutations = Collections.emptyList();
+
+    key.write(dos);
+    value.write(dos);
+
+    dos.close();
+
+    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+    Mutation m = new Mutation("m");
+    m.put("m", "m", "M");
+    bw.addMutation(m);
+    bw.close();
+
+    log.info("State of metadata table after inserting a record");
+
+    Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+    for (Entry<Key,Value> entry : s) {
+      System.out.println(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
+    }
+
+    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    s.setRange(MetadataSchema.ReplicationSection.getRange());
+    for (Entry<Key,Value> entry : s) {
+      System.out.println(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
+    }
+
+    log.info("Offline'ing table");
+
+    conn.tableOperations().offline(tableName, true);
+
+    // Add our fake WAL to the log column for this table
+    String walUri = tserverWal.toURI().toString();
+    KeyExtent extent = new KeyExtent(new Text(tableId), null, null);
+    bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+    m = new Mutation(extent.getMetadataEntry());
+    m.put(MetadataSchema.TabletsSection.LogColumnFamily.NAME, new Text("localhost:12345/" + walUri), new Value((walUri + "|1").getBytes(UTF_8)));
+    bw.addMutation(m);
+
+    // Add a replication entry for our fake WAL
+    m = new Mutation(MetadataSchema.ReplicationSection.getRowPrefix() + new Path(walUri).toString());
+    m.put(MetadataSchema.ReplicationSection.COLF, new Text(tableId), new Value(StatusUtil.fileCreated(System.currentTimeMillis()).toByteArray()));
+    bw.addMutation(m);
+    bw.close();
+
+    log.info("State of metadata after injecting WAL manually");
+
+    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+    for (Entry<Key,Value> entry : s) {
+      log.info(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
+    }
+
+    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    s.setRange(MetadataSchema.ReplicationSection.getRange());
+    for (Entry<Key,Value> entry : s) {
+      log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
+    }
+
+    log.info("Bringing table online");
+    conn.tableOperations().online(tableName, true);
+
+    Assert.assertEquals(1, Iterables.size(conn.createScanner(tableName, Authorizations.EMPTY)));
+
+    log.info("Table has performed recovery, state of metadata:");
+
+    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+    for (Entry<Key,Value> entry : s) {
+      log.info(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
+    }
+
+    s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    s.setRange(MetadataSchema.ReplicationSection.getRange());
+    for (Entry<Key,Value> entry : s) {
+      Status status = Status.parseFrom(entry.getValue().get());
+      log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(status));
+      Assert.assertFalse("Status record was closed and it should not be", status.getClosed());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/server/security/SystemCredentialsIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/server/security/SystemCredentialsIT.java b/test/src/main/java/org/apache/accumulo/test/server/security/SystemCredentialsIT.java
new file mode 100644
index 0000000..9752916
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/server/security/SystemCredentialsIT.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.server.security;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.junit.Test;
+
+public class SystemCredentialsIT extends ConfigurableMacBase {
+
+  private static final int FAIL_CODE = 7, BAD_PASSWD_FAIL_CODE = 8;
+
+  @Override
+  protected int defaultTimeoutSeconds() {
+    return 1 * 60;
+  }
+
+  @Test
+  public void testSystemCredentials() throws Exception {
+    assertEquals(0, exec(SystemCredentialsIT.class, "good", getCluster().getZooKeepers()).waitFor());
+    assertEquals(FAIL_CODE, exec(SystemCredentialsIT.class, "bad", getCluster().getZooKeepers()).waitFor());
+    assertEquals(BAD_PASSWD_FAIL_CODE, exec(SystemCredentialsIT.class, "bad_password", getCluster().getZooKeepers()).waitFor());
+  }
+
+  public static void main(final String[] args) throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
+    Credentials creds = null;
+    if (args.length < 2)
+      throw new RuntimeException("Incorrect usage; expected to be run by test only");
+    if (args[0].equals("bad")) {
+      Instance inst = new Instance() {
+
+        @Deprecated
+        @Override
+        public void setConfiguration(AccumuloConfiguration conf) {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public int getZooKeepersSessionTimeOut() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getZooKeepers() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getRootTabletLocation() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<String> getMasterLocations() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getInstanceName() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getInstanceID() {
+          return SystemCredentials.class.getName();
+        }
+
+        @Override
+        public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public AccumuloConfiguration getConfiguration() {
+          throw new UnsupportedOperationException();
+        }
+
+      };
+      creds = SystemCredentials.get(inst);
+    } else if (args[0].equals("good")) {
+      creds = SystemCredentials.get(HdfsZooInstance.getInstance());
+    } else if (args[0].equals("bad_password")) {
+      Instance inst = new Instance() {
+
+        @Override
+        public int getZooKeepersSessionTimeOut() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getZooKeepers() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getRootTabletLocation() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<String> getMasterLocations() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getInstanceName() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getInstanceID() {
+          return SystemCredentials.class.getName();
+        }
+
+        @Override
+        public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public AccumuloConfiguration getConfiguration() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Deprecated
+        @Override
+        public void setConfiguration(AccumuloConfiguration conf) {
+          throw new UnsupportedOperationException();
+        }
+
+      };
+      creds = new SystemCredentials(inst, "!SYSTEM", new PasswordToken("fake"));
+    } else {
+      throw new RuntimeException("Incorrect usage; expected to be run by test only");
+    }
+    Instance instance = HdfsZooInstance.getInstance();
+    Connector conn;
+    try {
+      conn = instance.getConnector(creds.getPrincipal(), creds.getToken());
+    } catch (AccumuloSecurityException e) {
+      e.printStackTrace(System.err);
+      System.exit(BAD_PASSWD_FAIL_CODE);
+      return;
+    }
+    try {
+      Scanner scan = conn.createScanner(RootTable.NAME, Authorizations.EMPTY);
+      for (Entry<Key,Value> e : scan) {
+        e.hashCode();
+      }
+    } catch (RuntimeException e) {
+      // catch the runtime exception from the scanner iterator
+      if (e.getCause() instanceof AccumuloSecurityException
+          && ((AccumuloSecurityException) e.getCause()).getSecurityErrorCode() == SecurityErrorCode.BAD_CREDENTIALS) {
+        e.printStackTrace(System.err);
+        System.exit(FAIL_CODE);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
new file mode 100644
index 0000000..f7f250a
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.start;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.file.rfile.PrintInfo;
+import org.apache.accumulo.core.util.Classpath;
+import org.apache.accumulo.core.util.CreateToken;
+import org.apache.accumulo.core.util.Help;
+import org.apache.accumulo.core.util.Jar;
+import org.apache.accumulo.core.util.Version;
+import org.apache.accumulo.gc.GCExecutable;
+import org.apache.accumulo.gc.SimpleGarbageCollector;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.master.MasterExecutable;
+import org.apache.accumulo.minicluster.MiniAccumuloRunner;
+import org.apache.accumulo.minicluster.impl.MiniClusterExecutable;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.MonitorExecutable;
+import org.apache.accumulo.proxy.Proxy;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.server.util.Info;
+import org.apache.accumulo.server.util.LoginProperties;
+import org.apache.accumulo.server.util.ZooKeeperMain;
+import org.apache.accumulo.shell.Shell;
+import org.apache.accumulo.start.Main;
+import org.apache.accumulo.start.spi.KeywordExecutable;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.accumulo.tracer.TracerExecutable;
+import org.apache.accumulo.tserver.TServerExecutable;
+import org.apache.accumulo.tserver.TabletServer;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KeywordStartIT {
+
+  private final Logger log = LoggerFactory.getLogger(getClass());
+
+  @Test
+  public void testKeywordsMatch() throws IOException {
+    for (Entry<String,KeywordExecutable> entry : Main.getExecutables(getClass().getClassLoader()).entrySet()) {
+      assertEquals(entry.getKey(), entry.getValue().keyword());
+    }
+  }
+
+  @Test
+  public void testCheckDuplicates() {
+    NoOp one = new NoOp("one");
+    NoOp anotherOne = new NoOp("another");
+    NoOp two = new NoOp("two");
+    NoOp three = new NoOp("three");
+    List<NoOp> services = Arrays.asList(one, three, two, two, three, three, anotherOne);
+    assertEquals(7, services.size());
+    Map<String,KeywordExecutable> results = Main.checkDuplicates(services);
+    assertTrue(results.containsKey(one.keyword()));
+    assertTrue(results.containsKey(anotherOne.keyword()));
+    assertFalse(results.containsKey(two.keyword()));
+    assertFalse(results.containsKey(three.keyword()));
+    assertEquals(2, results.size());
+  }
+
+  // Note: this test may fail in Eclipse, if the services files haven't been generated by the AutoService annotation processor
+  @Test
+  public void testExpectedClasses() throws IOException {
+    TreeMap<String,Class<? extends KeywordExecutable>> expectSet = new TreeMap<>();
+    expectSet.put("admin", Admin.class);
+    expectSet.put("classpath", Classpath.class);
+    expectSet.put("create-token", CreateToken.class);
+    expectSet.put("gc", GCExecutable.class);
+    expectSet.put("help", Help.class);
+    expectSet.put("info", Info.class);
+    expectSet.put("init", Initialize.class);
+    expectSet.put("jar", Jar.class);
+    expectSet.put("login-info", LoginProperties.class);
+    expectSet.put("master", MasterExecutable.class);
+    expectSet.put("minicluster", MiniClusterExecutable.class);
+    expectSet.put("monitor", MonitorExecutable.class);
+    expectSet.put("proxy", Proxy.class);
+    expectSet.put("rfile-info", PrintInfo.class);
+    expectSet.put("shell", Shell.class);
+    expectSet.put("tracer", TracerExecutable.class);
+    expectSet.put("tserver", TServerExecutable.class);
+    expectSet.put("version", Version.class);
+    expectSet.put("zookeeper", ZooKeeperMain.class);
+
+    Iterator<Entry<String,Class<? extends KeywordExecutable>>> expectIter = expectSet.entrySet().iterator();
+    TreeMap<String,KeywordExecutable> actualSet = new TreeMap<>(Main.getExecutables(getClass().getClassLoader()));
+    Iterator<Entry<String,KeywordExecutable>> actualIter = actualSet.entrySet().iterator();
+    Entry<String,Class<? extends KeywordExecutable>> expected;
+    Entry<String,KeywordExecutable> actual;
+    while (expectIter.hasNext() && actualIter.hasNext()) {
+      expected = expectIter.next();
+      actual = actualIter.next();
+      assertEquals(expected.getKey(), actual.getKey());
+      assertEquals(expected.getValue(), actual.getValue().getClass());
+    }
+    boolean moreExpected = expectIter.hasNext();
+    if (moreExpected) {
+      while (expectIter.hasNext()) {
+        log.warn("Missing class for keyword '" + expectIter.next() + "'");
+      }
+    }
+    assertFalse("Missing expected classes", moreExpected);
+    boolean moreActual = actualIter.hasNext();
+    if (moreActual) {
+      while (actualIter.hasNext()) {
+        log.warn("Extra class found with keyword '" + actualIter.next() + "'");
+      }
+    }
+    assertFalse("Found additional unexpected classes", moreActual);
+  }
+
+  @Test
+  public void checkHasMain() {
+    assertFalse("Sanity check for test failed. Somehow the test class has a main method", hasMain(this.getClass()));
+
+    HashSet<Class<?>> expectSet = new HashSet<>();
+    expectSet.add(Admin.class);
+    expectSet.add(CreateToken.class);
+    expectSet.add(Info.class);
+    expectSet.add(Initialize.class);
+    expectSet.add(LoginProperties.class);
+    expectSet.add(Master.class);
+    expectSet.add(MiniAccumuloRunner.class);
+    expectSet.add(Monitor.class);
+    expectSet.add(PrintInfo.class);
+    expectSet.add(Proxy.class);
+    expectSet.add(Shell.class);
+    expectSet.add(SimpleGarbageCollector.class);
+    expectSet.add(TabletServer.class);
+    expectSet.add(TraceServer.class);
+    expectSet.add(ZooKeeperMain.class);
+
+    for (Class<?> c : expectSet) {
+      assertTrue("Class " + c.getName() + " is missing a main method!", hasMain(c));
+    }
+
+  }
+
+  private static boolean hasMain(Class<?> classToCheck) {
+    Method main;
+    try {
+      main = classToCheck.getMethod("main", new String[0].getClass());
+    } catch (NoSuchMethodException e) {
+      return false;
+    }
+    return main != null && Modifier.isPublic(main.getModifiers()) && Modifier.isStatic(main.getModifiers());
+  }
+
+  private static class NoOp implements KeywordExecutable {
+
+    private final String kw;
+
+    public NoOp(String kw) {
+      this.kw = kw;
+    }
+
+    @Override
+    public String keyword() {
+      return kw;
+    }
+
+    @Override
+    public void execute(String[] args) throws Exception {}
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/util/CertUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/util/CertUtils.java b/test/src/main/java/org/apache/accumulo/test/util/CertUtils.java
new file mode 100644
index 0000000..2345ea7
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/util/CertUtils.java
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.util;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.Security;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.util.Calendar;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.commons.io.FileExistsException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x500.style.IETFUtils;
+import org.bouncycastle.asn1.x500.style.RFC4519Style;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.cert.CertIOException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils;
+import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.jce.provider.X509CertificateObject;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.google.common.base.Predicate;
+
+public class CertUtils {
+  private static final Logger log = LoggerFactory.getLogger(CertUtils.class);
+  static {
+    Security.addProvider(new BouncyCastleProvider());
+  }
+
+  static class Opts extends Help {
+    @Parameter(description = "generate-all | generate-local | generate-self-trusted", required = true, arity = 1)
+    List<String> operation = null;
+
+    @Parameter(names = {"--local-keystore"}, description = "Target path for generated keystore")
+    String localKeystore = null;
+
+    @Parameter(names = {"--root-keystore"}, description = "Path to root truststore, generated with generate-all, or used for signing with generate-local")
+    String rootKeystore = null;
+
+    @Parameter(names = {"--root-truststore"}, description = "Target path for generated public root truststore")
+    String truststore = null;
+
+    @Parameter(names = {"--keystore-type"}, description = "Type of keystore file to use")
+    String keystoreType = "JKS";
+
+    @Parameter(names = {"--root-keystore-password"}, description = "Password for root keystore, falls back to --keystore-password if not provided")
+    String rootKeystorePassword = null;
+
+    @Parameter(
+        names = {"--keystore-password"},
+        description = "Password used to encrypt keystores.  If omitted, the instance-wide secret will be used.  If specified, the password must also be explicitly configured in Accumulo.")
+    String keystorePassword = null;
+
+    @Parameter(names = {"--truststore-password"}, description = "Password used to encrypt the truststore. If omitted, empty password is used")
+    String truststorePassword = "";
+
+    @Parameter(names = {"--key-name-prefix"}, description = "Prefix for names of generated keys")
+    String keyNamePrefix = CertUtils.class.getSimpleName();
+
+    @Parameter(names = {"--issuer-rdn"}, description = "RDN string for issuer, for example: 'c=US,o=My Organization,cn=My Name'")
+    String issuerDirString = "o=Apache Accumulo";
+
+    @Parameter(names = "--site-file", description = "Load configuration from the given site file")
+    public String siteFile = null;
+
+    @Parameter(names = "--signing-algorithm", description = "Algorithm used to sign certificates")
+    public String signingAlg = "SHA256WITHRSA";
+
+    @Parameter(names = "--encryption-algorithm", description = "Algorithm used to encrypt private keys")
+    public String encryptionAlg = "RSA";
+
+    @Parameter(names = "--keysize", description = "Key size used by encryption algorithm")
+    public int keysize = 2048;
+
+    public AccumuloConfiguration getConfiguration() {
+      if (siteFile == null) {
+        return SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
+      } else {
+        return new AccumuloConfiguration() {
+          Configuration xml = new Configuration();
+          {
+            xml.addResource(new Path(siteFile));
+          }
+
+          @Override
+          public Iterator<Entry<String,String>> iterator() {
+            TreeMap<String,String> map = new TreeMap<String,String>();
+            for (Entry<String,String> props : DefaultConfiguration.getInstance())
+              map.put(props.getKey(), props.getValue());
+            for (Entry<String,String> props : xml)
+              map.put(props.getKey(), props.getValue());
+            return map.entrySet().iterator();
+          }
+
+          @Override
+          public String get(Property property) {
+            String value = xml.get(property.getKey());
+            if (value != null)
+              return value;
+            return DefaultConfiguration.getInstance().get(property);
+          }
+
+          @Override
+          public void getProperties(Map<String,String> props, Predicate<String> filter) {
+            for (Entry<String,String> entry : this)
+              if (filter.apply(entry.getKey()))
+                props.put(entry.getKey(), entry.getValue());
+          }
+        };
+      }
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    Opts opts = new Opts();
+    opts.parseArgs(CertUtils.class.getName(), args);
+    String operation = opts.operation.get(0);
+
+    String keyPassword = opts.keystorePassword;
+    if (keyPassword == null)
+      keyPassword = getDefaultKeyPassword();
+
+    String rootKeyPassword = opts.rootKeystorePassword;
+    if (rootKeyPassword == null) {
+      rootKeyPassword = keyPassword;
+    }
+
+    CertUtils certUtils = new CertUtils(opts.keystoreType, opts.issuerDirString, opts.encryptionAlg, opts.keysize, opts.signingAlg);
+
+    if ("generate-all".equals(operation)) {
+      certUtils.createAll(new File(opts.rootKeystore), new File(opts.localKeystore), new File(opts.truststore), opts.keyNamePrefix, rootKeyPassword,
+          keyPassword, opts.truststorePassword);
+    } else if ("generate-local".equals(operation)) {
+      certUtils.createSignedCert(new File(opts.localKeystore), opts.keyNamePrefix + "-local", keyPassword, opts.rootKeystore, rootKeyPassword);
+    } else if ("generate-self-trusted".equals(operation)) {
+      certUtils.createSelfSignedCert(new File(opts.truststore), opts.keyNamePrefix + "-selfTrusted", keyPassword);
+    } else {
+      JCommander jcommander = new JCommander(opts);
+      jcommander.setProgramName(CertUtils.class.getName());
+      jcommander.usage();
+      System.err.println("Unrecognized operation: " + opts.operation);
+      System.exit(0);
+    }
+  }
+
+  private static String getDefaultKeyPassword() {
+    return SiteConfiguration.getInstance(DefaultConfiguration.getInstance()).get(Property.INSTANCE_SECRET);
+  }
+
+  private String issuerDirString;
+  private String keystoreType;
+  private String encryptionAlgorithm;
+  private int keysize;
+  private String signingAlgorithm;
+
+  public CertUtils(String keystoreType, String issuerDirString, String encryptionAlgorithm, int keysize, String signingAlgorithm) {
+    super();
+    this.keystoreType = keystoreType;
+    this.issuerDirString = issuerDirString;
+    this.encryptionAlgorithm = encryptionAlgorithm;
+    this.keysize = keysize;
+    this.signingAlgorithm = signingAlgorithm;
+  }
+
+  public void createAll(File rootKeystoreFile, File localKeystoreFile, File trustStoreFile, String keyNamePrefix, String rootKeystorePassword,
+      String keystorePassword, String truststorePassword) throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException,
+      OperatorCreationException, AccumuloSecurityException, NoSuchProviderException, UnrecoverableKeyException, FileNotFoundException {
+    createSelfSignedCert(rootKeystoreFile, keyNamePrefix + "-root", rootKeystorePassword);
+    createSignedCert(localKeystoreFile, keyNamePrefix + "-local", keystorePassword, rootKeystoreFile.getAbsolutePath(), rootKeystorePassword);
+    createPublicCert(trustStoreFile, keyNamePrefix + "-public", rootKeystoreFile.getAbsolutePath(), rootKeystorePassword, truststorePassword);
+  }
+
+  public void createPublicCert(File targetKeystoreFile, String keyName, String rootKeystorePath, String rootKeystorePassword, String truststorePassword)
+      throws NoSuchAlgorithmException, CertificateException, FileNotFoundException, IOException, KeyStoreException, UnrecoverableKeyException {
+    KeyStore signerKeystore = KeyStore.getInstance(keystoreType);
+    char[] signerPasswordArray = rootKeystorePassword.toCharArray();
+    try (FileInputStream fis = new FileInputStream(rootKeystorePath)) {
+      signerKeystore.load(fis, signerPasswordArray);
+    }
+    Certificate rootCert = findCert(signerKeystore);
+
+    KeyStore keystore = KeyStore.getInstance(keystoreType);
+    keystore.load(null, null);
+    keystore.setCertificateEntry(keyName + "Cert", rootCert);
+    try (FileOutputStream fos = new FileOutputStream(targetKeystoreFile)) {
+      keystore.store(fos, truststorePassword.toCharArray());
+    }
+  }
+
+  public void createSignedCert(File targetKeystoreFile, String keyName, String keystorePassword, String signerKeystorePath, String signerKeystorePassword)
+      throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException, OperatorCreationException, AccumuloSecurityException,
+      UnrecoverableKeyException, NoSuchProviderException {
+    KeyStore signerKeystore = KeyStore.getInstance(keystoreType);
+    char[] signerPasswordArray = signerKeystorePassword.toCharArray();
+    try (FileInputStream fis = new FileInputStream(signerKeystorePath)) {
+      signerKeystore.load(fis, signerPasswordArray);
+    }
+    Certificate signerCert = findCert(signerKeystore);
+    PrivateKey signerKey = findPrivateKey(signerKeystore, signerPasswordArray);
+
+    KeyPair kp = generateKeyPair();
+    X509CertificateObject cert = generateCert(keyName, kp, false, signerCert.getPublicKey(), signerKey);
+
+    char[] password = keystorePassword.toCharArray();
+    KeyStore keystore = KeyStore.getInstance(keystoreType);
+    keystore.load(null, null);
+    keystore.setCertificateEntry(keyName + "Cert", cert);
+    keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new Certificate[] {cert, signerCert});
+    try (FileOutputStream fos = new FileOutputStream(targetKeystoreFile)) {
+      keystore.store(fos, password);
+    }
+  }
+
+  public void createSelfSignedCert(File targetKeystoreFile, String keyName, String keystorePassword) throws KeyStoreException, CertificateException,
+      NoSuchAlgorithmException, IOException, OperatorCreationException, AccumuloSecurityException, NoSuchProviderException {
+    if (targetKeystoreFile.exists()) {
+      throw new FileExistsException(targetKeystoreFile);
+    }
+
+    KeyPair kp = generateKeyPair();
+
+    X509CertificateObject cert = generateCert(keyName, kp, true, kp.getPublic(), kp.getPrivate());
+
+    char[] password = keystorePassword.toCharArray();
+    KeyStore keystore = KeyStore.getInstance(keystoreType);
+    keystore.load(null, null);
+    keystore.setCertificateEntry(keyName + "Cert", cert);
+    keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new Certificate[] {cert});
+    try (FileOutputStream fos = new FileOutputStream(targetKeystoreFile)) {
+      keystore.store(fos, password);
+    }
+  }
+
+  private KeyPair generateKeyPair() throws NoSuchAlgorithmException, NoSuchProviderException {
+    KeyPairGenerator gen = KeyPairGenerator.getInstance(encryptionAlgorithm);
+    gen.initialize(keysize);
+    return gen.generateKeyPair();
+  }
+
+  private X509CertificateObject generateCert(String keyName, KeyPair kp, boolean isCertAuthority, PublicKey signerPublicKey, PrivateKey signerPrivateKey)
+      throws IOException, CertIOException, OperatorCreationException, CertificateException, NoSuchAlgorithmException {
+    Calendar startDate = Calendar.getInstance();
+    Calendar endDate = Calendar.getInstance();
+    endDate.add(Calendar.YEAR, 100);
+
+    BigInteger serialNumber = BigInteger.valueOf((startDate.getTimeInMillis()));
+    X500Name issuer = new X500Name(IETFUtils.rDNsFromString(issuerDirString, RFC4519Style.INSTANCE));
+    JcaX509v3CertificateBuilder certGen = new JcaX509v3CertificateBuilder(issuer, serialNumber, startDate.getTime(), endDate.getTime(), issuer, kp.getPublic());
+    JcaX509ExtensionUtils extensionUtils = new JcaX509ExtensionUtils();
+    certGen.addExtension(Extension.subjectKeyIdentifier, false, extensionUtils.createSubjectKeyIdentifier(kp.getPublic()));
+    certGen.addExtension(Extension.basicConstraints, false, new BasicConstraints(isCertAuthority));
+    certGen.addExtension(Extension.authorityKeyIdentifier, false, extensionUtils.createAuthorityKeyIdentifier(signerPublicKey));
+    if (isCertAuthority) {
+      certGen.addExtension(Extension.keyUsage, true, new KeyUsage(KeyUsage.keyCertSign));
+    }
+    X509CertificateHolder cert = certGen.build(new JcaContentSignerBuilder(signingAlgorithm).build(signerPrivateKey));
+    return new X509CertificateObject(cert.toASN1Structure());
+  }
+
+  static Certificate findCert(KeyStore keyStore) throws KeyStoreException {
+    Enumeration<String> aliases = keyStore.aliases();
+    Certificate cert = null;
+    while (aliases.hasMoreElements()) {
+      String alias = aliases.nextElement();
+      if (keyStore.isCertificateEntry(alias)) {
+        if (cert == null) {
+          cert = keyStore.getCertificate(alias);
+        } else {
+          log.warn("Found multiple certificates in keystore.  Ignoring " + alias);
+        }
+      }
+    }
+    if (cert == null) {
+      throw new KeyStoreException("Could not find cert in keystore");
+    }
+    return cert;
+  }
+
+  static PrivateKey findPrivateKey(KeyStore keyStore, char[] keystorePassword) throws UnrecoverableKeyException, KeyStoreException, NoSuchAlgorithmException {
+    Enumeration<String> aliases = keyStore.aliases();
+    PrivateKey key = null;
+    while (aliases.hasMoreElements()) {
+      String alias = aliases.nextElement();
+      if (keyStore.isKeyEntry(alias)) {
+        if (key == null) {
+          key = (PrivateKey) keyStore.getKey(alias, keystorePassword);
+        } else {
+          log.warn("Found multiple keys in keystore.  Ignoring " + alias);
+        }
+      }
+    }
+    if (key == null) {
+      throw new KeyStoreException("Could not find private key in keystore");
+    }
+    return key;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/FooConstraint.jar
----------------------------------------------------------------------
diff --git a/test/src/main/resources/FooConstraint.jar b/test/src/main/resources/FooConstraint.jar
new file mode 100644
index 0000000..14673da
Binary files /dev/null and b/test/src/main/resources/FooConstraint.jar differ

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/FooFilter.jar
----------------------------------------------------------------------
diff --git a/test/src/main/resources/FooFilter.jar b/test/src/main/resources/FooFilter.jar
new file mode 100644
index 0000000..ef30cbc
Binary files /dev/null and b/test/src/main/resources/FooFilter.jar differ

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/TestCombinerX.jar
----------------------------------------------------------------------
diff --git a/test/src/main/resources/TestCombinerX.jar b/test/src/main/resources/TestCombinerX.jar
new file mode 100644
index 0000000..849e447
Binary files /dev/null and b/test/src/main/resources/TestCombinerX.jar differ

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/TestCombinerY.jar
----------------------------------------------------------------------
diff --git a/test/src/main/resources/TestCombinerY.jar b/test/src/main/resources/TestCombinerY.jar
new file mode 100644
index 0000000..30da0cbf
Binary files /dev/null and b/test/src/main/resources/TestCombinerY.jar differ

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/TestCompactionStrat.jar
----------------------------------------------------------------------
diff --git a/test/src/main/resources/TestCompactionStrat.jar b/test/src/main/resources/TestCompactionStrat.jar
new file mode 100644
index 0000000..3daa16e
Binary files /dev/null and b/test/src/main/resources/TestCompactionStrat.jar differ

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/conf/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/test/src/main/resources/conf/accumulo-site.xml b/test/src/main/resources/conf/accumulo-site.xml
new file mode 100644
index 0000000..e1f0e70
--- /dev/null
+++ b/test/src/main/resources/conf/accumulo-site.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+	<!--
+  Put your site-specific accumulo configurations here.
+
+  The available configuration values along with their defaults
+  are documented in docs/config.html
+
+  Unless you are simply testing at your workstation, you will most 
+  definitely need to change the three entries below.
+	-->
+
+  <property>
+    <name>instance.zookeeper.host</name>
+    <value>localhost:2181</value>
+    <description>comma separated list of zookeeper servers</description>
+  </property>
+
+  <property>
+    <name>logger.dir.walog</name>
+    <value>walogs</value>
+    <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.
+    </description>
+  </property>
+
+  <property>
+    <name>instance.secret</name>
+    <value>DEFAULT</value>
+    <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+      Change it before initialization. To
+      change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+      and then update this file.
+    </description>
+  </property>
+
+  <property>
+    <name>tserver.memory.maps.max</name>
+    <value>80M</value>
+  </property>
+
+  <property>
+    <name>tserver.cache.data.size</name>
+    <value>7M</value>
+  </property>
+
+  <property>
+    <name>tserver.cache.index.size</name>
+    <value>20M</value>
+  </property>
+
+  <property>
+    <name>trace.password</name>
+      <!-- 
+        change this to the root user's password, and/or change the user below 
+       -->
+    <value>secret</value>
+  </property>
+
+  <property>
+    <name>trace.user</name>
+    <value>root</value>
+  </property>
+
+  <property>
+    <name>tserver.sort.buffer.size</name>
+    <value>50M</value>
+  </property>
+
+  <property>
+    <name>tserver.walog.max.size</name>
+    <value>100M</value>
+  </property>
+
+  <property>
+    <name>general.classpaths</name>
+    <!--
+       Add the following for hadoop-2.0
+       $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+       $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+       $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+       $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+       $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+    -->
+    <value>
+      $ACCUMULO_HOME/server/target/classes/,
+      $ACCUMULO_HOME/lib/accumulo-server.jar,
+      $ACCUMULO_HOME/core/target/classes/,
+      $ACCUMULO_HOME/lib/accumulo-core.jar,
+      $ACCUMULO_HOME/start/target/classes/,
+      $ACCUMULO_HOME/lib/accumulo-start.jar,
+      $ACCUMULO_HOME/fate/target/classes/,
+      $ACCUMULO_HOME/lib/accumulo-fate.jar,
+      $ACCUMULO_HOME/proxy/target/classes/,
+      $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+      $ACCUMULO_HOME/lib/[^.].*.jar,
+      $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+      $HADOOP_CONF_DIR,
+      $HADOOP_PREFIX/[^.].*.jar,
+      $HADOOP_PREFIX/lib/[^.].*.jar,
+    </value>
+    <description>Classpaths that accumulo checks for updates and class files.
+      When using the Security Manager, please remove the ".../target/classes/" values.
+    </description>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/resources/conf/generic_logger.xml
----------------------------------------------------------------------
diff --git a/test/src/main/resources/conf/generic_logger.xml b/test/src/main/resources/conf/generic_logger.xml
new file mode 100644
index 0000000..db79efe
--- /dev/null
+++ b/test/src/main/resources/conf/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+  <!-- Write out everything at the DEBUG level to the debug log -->
+  <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+     <param name="File"           value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+     <param name="MaxFileSize"    value="1000MB"/>
+     <param name="MaxBackupIndex" value="10"/>
+     <param name="Threshold"      value="DEBUG"/>
+     <layout class="org.apache.log4j.PatternLayout">
+       <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+     </layout>
+  </appender>
+
+  <!--  Write out INFO and higher to the regular log -->
+  <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+     <param name="File"           value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+     <param name="MaxFileSize"    value="1000MB"/>
+     <param name="MaxBackupIndex" value="10"/>
+     <param name="Threshold"      value="INFO"/>
+     <layout class="org.apache.log4j.PatternLayout">
+       <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+     </layout>
+  </appender>
+
+  <!-- Send all logging data to a centralized logger -->
+  <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+     <param name="remoteHost"     value="${org.apache.accumulo.core.host.log}"/>
+     <param name="port"           value="${org.apache.accumulo.core.host.log.port}"/>
+     <param name="application"    value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+     <param name="Threshold"      value="WARN"/>
+  </appender>
+
+  <!--  If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+  <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+     <appender-ref ref="N1" />
+  </appender>
+
+  <!-- Log accumulo events to the debug, normal and remote logs. -->
+  <logger name="org.apache.accumulo" additivity="false">
+     <level value="DEBUG"/>
+     <appender-ref ref="A2" />
+     <appender-ref ref="A3" />
+     <appender-ref ref="ASYNC" />
+  </logger>
+
+  <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+     <level value="INFO"/>
+  </logger>
+
+  <logger name="org.mortbay.log">
+     <level value="WARN"/>
+  </logger>
+
+  <logger name="org.apache.zookeeper">
+     <level value="ERROR"/>
+  </logger>
+
+  <!-- Log non-accumulo events to the debug and normal logs. -->
+  <root>
+     <level value="INFO"/>
+     <appender-ref ref="A2" />
+     <appender-ref ref="A3" />
+  </root>
+
+</log4j:configuration>


Mime
View raw message