Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F2E1910AEA for ; Tue, 19 Nov 2013 19:58:14 +0000 (UTC) Received: (qmail 55851 invoked by uid 500); 19 Nov 2013 19:58:14 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 55763 invoked by uid 500); 19 Nov 2013 19:58:14 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 55740 invoked by uid 99); 19 Nov 2013 19:58:14 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Nov 2013 19:58:14 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 26B02883127; Tue, 19 Nov 2013 19:58:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ecn@apache.org To: commits@accumulo.apache.org Date: Tue, 19 Nov 2013 19:58:14 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/3] ACCUMULO-1009 Updated Branches: refs/heads/master b732722f8 -> 7038755be http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java index f7e1146..4fe3d3f 100644 --- a/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java +++ b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterTest.java @@ -34,13 +34,13 @@ import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.impl.MultiTableBatchWriterImpl; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.ClientConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.Credentials; -import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.minicluster.MiniAccumuloConfig; import org.junit.AfterClass; @@ -48,7 +48,6 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.mortbay.jetty.security.Credential; import com.google.common.collect.Maps; @@ -69,11 +68,11 @@ public class MultiTableBatchWriterTest { public static void tearDownAfterClass() throws Exception { cluster.stop(); folder.delete(); - } + } @Test public void testTableRenameDataValidation() throws Exception { - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -141,7 +140,7 @@ public class MultiTableBatchWriterTest { @Test public void testTableRenameSameWriters() throws Exception { - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -203,7 +202,7 @@ public class MultiTableBatchWriterTest { @Test public void testTableRenameNewWriters() throws Exception { - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -285,7 +284,7 @@ public class MultiTableBatchWriterTest { @Test public void testTableRenameNewWritersNoCaching() throws Exception { - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -334,7 +333,7 @@ public class MultiTableBatchWriterTest { @Test public void testTableDelete() throws Exception { - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -391,8 +390,7 @@ public class MultiTableBatchWriterTest { @Test public void testOfflineTable() throws Exception { - - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -448,8 +446,7 @@ public class MultiTableBatchWriterTest { @Test public void testOfflineTableWithCache() throws Exception { - - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); @@ -508,8 +505,7 @@ public class MultiTableBatchWriterTest { @Test public void testOfflineTableWithoutCache() throws Exception { - - ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + ZooKeeperInstance instance = new ZooKeeperInstance(new ClientConfiguration().withInstance(cluster.getInstanceName()).withZkHosts(cluster.getZooKeepers())); Connector connector = instance.getConnector("root", password); BatchWriterConfig config = new BatchWriterConfig(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java index dbf5f4c..ec906d6 100644 --- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java +++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java @@ -26,7 +26,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; import java.lang.reflect.Constructor; -import java.lang.reflect.Method; import java.util.Map.Entry; import jline.console.ConsoleReader; @@ -46,8 +45,6 @@ import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.shell.Shell; -import org.apache.accumulo.minicluster.MiniAccumuloCluster; -import org.apache.accumulo.minicluster.MiniAccumuloConfig; import org.apache.accumulo.test.functional.SimpleMacIT; import org.apache.accumulo.tracer.TraceServer; import org.apache.commons.io.FileUtils; @@ -59,9 +56,7 @@ import org.apache.hadoop.tools.DistCp; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; public class ShellServerIT extends SimpleMacIT { public static class TestOutputStream extends OutputStream { @@ -99,8 +94,6 @@ public class ShellServerIT extends SimpleMacIT { } } - private static String secret = "superSecret"; - public static MiniAccumuloCluster cluster; public static TestOutputStream output; public static StringInputStream input; public static Shell shell; @@ -152,28 +145,21 @@ public class ShellServerIT extends SimpleMacIT { @BeforeClass public static void setUpBeforeClass() throws Exception { - folder.create(); - MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret); - cluster = new MiniAccumuloCluster(cfg); - cluster.start(); - // history file is updated in $HOME - System.setProperty("HOME", folder.getRoot().getAbsolutePath()); + System.setProperty("HOME", getFolder().getAbsolutePath()); // start the shell output = new TestOutputStream(); input = new StringInputStream(); shell = new Shell(new ConsoleReader(input, output)); shell.setLogErrorsToConsole(); - shell.config("-u", "root", "-p", secret, "-z", cluster.getConfig().getInstanceName(), cluster.getConfig().getZooKeepers()); + shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(), + "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath()); exec("quit", true); shell.start(); shell.setExit(false); - // use reflection to call this method so it does not need to be made public - Method method = cluster.getClass().getDeclaredMethod("exec", Class.class, String[].class); - method.setAccessible(true); - traceProcess = (Process) method.invoke(cluster, TraceServer.class, new String[0]); + traceProcess = getStaticCluster().exec(TraceServer.class); // give the tracer some time to start UtilWaitThread.sleep(1000); @@ -181,14 +167,12 @@ public class ShellServerIT extends SimpleMacIT { @AfterClass public static void tearDownAfterClass() throws Exception { - cluster.stop(); traceProcess.destroy(); - folder.delete(); } @After - public void tearDown() throws Exception { - Connector c = cluster.getConnector("root", secret); + public void deleteTables() throws Exception { + Connector c = getConnector(); for (String table : c.tableOperations().list()) { if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace")) try { @@ -207,10 +191,10 @@ public class ShellServerIT extends SimpleMacIT { exec("addsplits row5", true); exec("config -t t -s table.split.threshold=345M", true); exec("offline t", true); - String export = "file://" + folder.newFolder().toString(); + String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString(); exec("exporttable -t t " + export, true); DistCp cp = newDistCp(); - String import_ = "file://" + folder.newFolder().toString(); + String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString(); cp.run(new String[] {"-f", export + "/distcp.txt", import_}); exec("importtable t2 " + import_, true); exec("config -t t2 -np", true, "345M", true); @@ -260,7 +244,7 @@ public class ShellServerIT extends SimpleMacIT { @Test(timeout = 30 * 1000) public void execfile() throws Exception { // execfile - File file = folder.newFile(); + File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder()); PrintWriter writer = new PrintWriter(file.getAbsolutePath()); writer.println("about"); writer.close(); @@ -326,7 +310,7 @@ public class ShellServerIT extends SimpleMacIT { exec("scan", true, "row1", true); exec("droptable -f t", true); exec("deleteuser xyzzy", false, "delete yourself", true); - input.set(secret + "\n" + secret + "\n"); + input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n"); exec("user root", true); exec("revoke -u xyzzy -s System.CREATE_TABLE", true); exec("revoke -u xyzzy -s System.GOOFY", false); @@ -598,10 +582,12 @@ public class ShellServerIT extends SimpleMacIT { public void importDirectory() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); - File importDir = folder.newFolder("import"); + File importDir = new File(getFolder(), "import"); + importDir.mkdir(); String even = new File(importDir, "even.rf").toString(); String odd = new File(importDir, "odd.rf").toString(); - File errorsDir = folder.newFolder("errors"); + File errorsDir = new File(getFolder(), "errors"); + errorsDir.mkdir(); fs.mkdirs(new Path(errorsDir.toString())); AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration(); FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf); @@ -751,7 +737,7 @@ public class ShellServerIT extends SimpleMacIT { @Override public void run() { try { - Connector connector = cluster.getConnector("root", secret); + Connector connector = getConnector(); Scanner s = connector.createScanner("t", Authorizations.EMPTY); for (@SuppressWarnings("unused") Entry kv : s) @@ -769,20 +755,26 @@ public class ShellServerIT extends SimpleMacIT { assertTrue(last.contains("RUNNING")); String parts[] = last.split("\\|"); assertEquals(13, parts.length); + String hostPortPattern = ".+:\\d+"; + String tserver = parts[0].trim(); + assertTrue(tserver.matches(hostPortPattern)); + assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver)); + String client = parts[1].trim(); + assertTrue(client.matches(hostPortPattern)); + // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...? thread.join(); exec("deletetable -f t", true); } - @Rule - public TemporaryFolder folder2 = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); - @Test(timeout = 30 * 1000) public void testPertableClasspath() throws Exception { - File fooFilterJar = folder2.newFile("FooFilter.jar"); + File fooFilterJar = File.createTempFile("FooFilter", ".jar"); FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar); + fooFilterJar.deleteOnExit(); - File fooConstraintJar = folder2.newFile("FooConstraint.jar"); + File fooConstraintJar = File.createTempFile("FooConstraint", ".jar"); FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar); + fooConstraintJar.deleteOnExit(); exec( "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(), @@ -832,7 +824,7 @@ public class ShellServerIT extends SimpleMacIT { @Test(timeout = 30 * 1000) public void badLogin() throws Exception { - input.set(secret + "\n"); + input.set(ROOT_PASSWORD + "\n"); String err = exec("user NoSuchUser", false); assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser")); } @@ -872,7 +864,7 @@ public class ShellServerIT extends SimpleMacIT { input.set("secret\n"); exec("user test_user", true); assertTrue(exec("whoami", true).contains("test_user")); - input.set(secret + "\n"); + input.set(ROOT_PASSWORD + "\n"); exec("user root", true); } @@ -886,11 +878,4 @@ public class ShellServerIT extends SimpleMacIT { exec("scan -t !METADATA -np -c file"); return output.get().split("\n").length - 1; } - - public static TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target/")); - - public MiniAccumuloCluster getCluster() { - return cluster; - } - } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java index d24b85b..f74b205 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/AbstractMacIT.java @@ -17,6 +17,7 @@ package org.apache.accumulo.test.functional; import java.io.File; +import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; @@ -25,7 +26,10 @@ import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.minicluster.MiniAccumuloCluster; +import org.apache.accumulo.minicluster.MiniAccumuloConfig; +import org.apache.accumulo.test.util.CertUtils; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.junit.Rule; @@ -81,8 +85,39 @@ public abstract class AbstractMacIT { return names; } + protected static void configureForEnvironment(MiniAccumuloConfig cfg, Class testClass, File folder) { + if ("true".equals(System.getProperty("org.apache.accumulo.test.functional.useSslForIT"))) { + configureForSsl(cfg, folder); + } + } + + protected static void configureForSsl(MiniAccumuloConfig cfg, File folder) { + Map siteConfig = cfg.getSiteConfig(); + if ("true".equals(siteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) { + // already enabled; don't mess with it + return; + } + + File sslDir = new File(folder, "ssl"); + sslDir.mkdirs(); + File rootKeystoreFile = new File(sslDir, "root-" + cfg.getInstanceName() + ".jks"); + File localKeystoreFile = new File(sslDir, "local-" + cfg.getInstanceName() + ".jks"); + File publicTruststoreFile = new File(sslDir, "public-" + cfg.getInstanceName() + ".jks"); + try { + new CertUtils(Property.RPC_SSL_KEYSTORE_TYPE.getDefaultValue(), "o=Apache Accumulo,cn=MiniAccumuloCluster", "RSA", 2048, "sha1WithRSAEncryption") + .createAll(rootKeystoreFile, localKeystoreFile, publicTruststoreFile, cfg.getInstanceName(), cfg.getRootPassword()); + } catch (Exception e) { + throw new RuntimeException("error creating MAC keystore", e); + } + + siteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true"); + siteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), localKeystoreFile.getAbsolutePath()); + siteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), cfg.getRootPassword()); + siteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), publicTruststoreFile.getAbsolutePath()); + cfg.setSiteConfig(siteConfig); + } + public abstract Connector getConnector() throws AccumuloException, AccumuloSecurityException; public abstract String rootPath(); - } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/AccumuloInputFormatIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/AccumuloInputFormatIT.java b/test/src/test/java/org/apache/accumulo/test/functional/AccumuloInputFormatIT.java index d38536a..3ebafff 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/AccumuloInputFormatIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/AccumuloInputFormatIT.java @@ -64,7 +64,7 @@ public class AccumuloInputFormatIT extends SimpleMacIT { @SuppressWarnings("deprecation") Job job = new Job(); AccumuloInputFormat.setInputTableName(job, table); - AccumuloInputFormat.setZooKeeperInstance(job, getConnector().getInstance().getInstanceName(), getConnector().getInstance().getZooKeepers()); + AccumuloInputFormat.setZooKeeperInstance(job, getStaticCluster().getClientConfig()); AccumuloInputFormat.setConnectorInfo(job, "root", new PasswordToken(ROOT_PASSWORD)); // split table @@ -100,7 +100,8 @@ public class AccumuloInputFormatIT extends SimpleMacIT { // auto adjust ranges ranges = new ArrayList(); - for (int i = 0; i < 5; i++) // overlapping ranges + for (int i = 0; i < 5; i++) + // overlapping ranges ranges.add(new Range(String.format("%09d", i), String.format("%09d", i + 2))); AccumuloInputFormat.setRanges(job, ranges); splits = inputFormat.getSplits(job); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java index 2fb5827..d86b704 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java @@ -16,7 +16,14 @@ */ package org.apache.accumulo.test.functional; +import java.io.IOException; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.test.TestIngest; import org.apache.accumulo.test.TestIngest.Opts; @@ -32,8 +39,11 @@ public class BulkIT extends SimpleMacIT { @Test(timeout = 4 * 60 * 1000) public void test() throws Exception { - Connector c = getConnector(); - String tableName = getTableNames(1)[0]; + runTest(getConnector(), getTableNames(1)[0]); + } + + static void runTest(Connector c, String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException, IOException, TableNotFoundException, + MutationsRejectedException { c.tableOperations().create(tableName); FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); String base = "target/accumulo-maven-plugin"; http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java index c3d3160..87f6bd7 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ConcurrencyIT.java @@ -20,11 +20,16 @@ import java.util.Collections; import java.util.EnumSet; import java.util.Map.Entry; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; @@ -75,13 +80,18 @@ public class ConcurrencyIT extends ConfigurableMacIT { @Test(timeout = 2 * 60 * 1000) public void run() throws Exception { Connector c = getConnector(); + runTest(c); + } + + static void runTest(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, + MutationsRejectedException, Exception, InterruptedException { c.tableOperations().create("cct"); IteratorSetting is = new IteratorSetting(10, SlowIterator.class); SlowIterator.setSleepTime(is, 50); c.tableOperations().attachIterator("cct", is, EnumSet.of(IteratorScope.minc, IteratorScope.majc)); c.tableOperations().setProperty("cct", Property.TABLE_MAJC_RATIO.getKey(), "1.0"); - BatchWriter bw = getConnector().createBatchWriter("cct", new BatchWriterConfig()); + BatchWriter bw = c.createBatchWriter("cct", new BatchWriterConfig()); for (int i = 0; i < 50; i++) { Mutation m = new Mutation(new Text(String.format("%06d", i))); m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes())); @@ -89,14 +99,14 @@ public class ConcurrencyIT extends ConfigurableMacIT { } bw.flush(); - ScanTask st0 = new ScanTask(getConnector(), 300); + ScanTask st0 = new ScanTask(c, 300); st0.start(); - ScanTask st1 = new ScanTask(getConnector(), 100); + ScanTask st1 = new ScanTask(c, 100); st1.start(); UtilWaitThread.sleep(50); - getConnector().tableOperations().flush("cct", null, null, true); + c.tableOperations().flush("cct", null, null, true); for (int i = 0; i < 50; i++) { Mutation m = new Mutation(new Text(String.format("%06d", i))); @@ -106,7 +116,7 @@ public class ConcurrencyIT extends ConfigurableMacIT { bw.flush(); - ScanTask st2 = new ScanTask(getConnector(), 100); + ScanTask st2 = new ScanTask(c, 100); st2.start(); st1.join(); @@ -117,11 +127,11 @@ public class ConcurrencyIT extends ConfigurableMacIT { if (st2.count != 50) throw new Exception("Thread 2 did not see 50, saw " + st2.count); - ScanTask st3 = new ScanTask(getConnector(), 150); + ScanTask st3 = new ScanTask(c, 150); st3.start(); UtilWaitThread.sleep(50); - getConnector().tableOperations().flush("cct", null, null, false); + c.tableOperations().flush("cct", null, null, false); st3.join(); if (st3.count != 50) http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java index 3f60f1d..21c2bb7 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java @@ -42,6 +42,7 @@ public class ConfigurableMacIT extends AbstractMacIT { public void setUp() throws Exception { MiniAccumuloConfig cfg = new MiniAccumuloConfig(createTestDir(this.getClass().getName()), ROOT_PASSWORD); configure(cfg); + configureForEnvironment(cfg, getClass(), createSharedTestDir(this.getClass().getName() + "-ssl")); cluster = new MiniAccumuloCluster(cfg); cluster.start(); } @@ -70,7 +71,7 @@ public class ConfigurableMacIT extends AbstractMacIT { } public String getMonitor() throws KeeperException, InterruptedException { - Instance instance = new ZooKeeperInstance(getCluster().getInstanceName(), getCluster().getZooKeepers()); + Instance instance = new ZooKeeperInstance(getCluster().getClientConfig()); return MonitorUtil.getLocation(instance); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java index 9e42e55..0867e73 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java @@ -18,18 +18,26 @@ package org.apache.accumulo.test.functional; import static org.junit.Assert.assertEquals; +import java.io.IOException; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Map.Entry; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.examples.simple.mapreduce.RowHash; +import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.hadoop.io.Text; import org.codehaus.plexus.util.Base64; import org.junit.Test; @@ -45,7 +53,11 @@ public class MapReduceIT extends ConfigurableMacIT { @Test(timeout = 60 * 1000) public void test() throws Exception { - Connector c = getConnector(); + runTest(getConnector(), getCluster()); + } + + static void runTest(Connector c, MiniAccumuloCluster cluster) throws AccumuloException, AccumuloSecurityException, TableExistsException, + TableNotFoundException, MutationsRejectedException, IOException, InterruptedException, NoSuchAlgorithmException { c.tableOperations().create(tablename); BatchWriter bw = c.createBatchWriter(tablename, new BatchWriterConfig()); for (int i = 0; i < 10; i++) { @@ -54,9 +66,8 @@ public class MapReduceIT extends ConfigurableMacIT { bw.addMutation(m); } bw.close(); - - Process hash = exec(RowHash.class, "-i", c.getInstance().getInstanceName(), "-z", c.getInstance().getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "-t", - tablename, "--column", input_cfcq); + Process hash = cluster.exec(RowHash.class, "-i", c.getInstance().getInstanceName(), "-z", c.getInstance().getZooKeepers(), "-u", "root", "-p", + ROOT_PASSWORD, "-t", tablename, "--column", input_cfcq); assertEquals(0, hash.waitFor()); Scanner s = c.createScanner(tablename, Authorizations.EMPTY); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/ScannerIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ScannerIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ScannerIT.java index 7913089..e364b46 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/ScannerIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ScannerIT.java @@ -46,55 +46,55 @@ public class ScannerIT extends SimpleMacIT { final String table = "table"; Connector c = getConnector(); c.tableOperations().create(table); - + BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig()); - + Mutation m = new Mutation("a"); for (int i = 0; i < 10; i++) { m.put(Integer.toString(i), "", ""); } - + bw.addMutation(m); bw.close(); - + Scanner s = c.createScanner(table, new Authorizations()); - + IteratorSetting cfg = new IteratorSetting(100, SlowIterator.class); SlowIterator.setSleepTime(cfg, 100l); s.addScanIterator(cfg); s.setReadaheadThreshold(5); s.setBatchSize(1); s.setRange(new Range()); - + Stopwatch sw = new Stopwatch(); Iterator> iterator = s.iterator(); - + sw.start(); while (iterator.hasNext()) { sw.stop(); - + // While we "do work" in the client, we should be fetching the next result UtilWaitThread.sleep(100l); iterator.next(); sw.start(); } sw.stop(); - + long millisWithWait = sw.elapsed(TimeUnit.MILLISECONDS); - + s = c.createScanner(table, new Authorizations()); s.addScanIterator(cfg); s.setRange(new Range()); s.setBatchSize(1); s.setReadaheadThreshold(0l); - + sw = new Stopwatch(); iterator = s.iterator(); - + sw.start(); while (iterator.hasNext()) { sw.stop(); - + // While we "do work" in the client, we should be fetching the next result UtilWaitThread.sleep(100l); iterator.next(); @@ -103,10 +103,10 @@ public class ScannerIT extends SimpleMacIT { sw.stop(); long millisWithNoWait = sw.elapsed(TimeUnit.MILLISECONDS); - + // The "no-wait" time should be much less than the "wait-time" - Assert.assertTrue("Expected less time to be taken with immediate readahead (" + millisWithNoWait - + ") than without immediate readahead (" + millisWithWait + ")", millisWithNoWait < millisWithWait); + Assert.assertTrue("Expected less time to be taken with immediate readahead (" + millisWithNoWait + ") than without immediate readahead (" + millisWithWait + + ")", millisWithNoWait < millisWithWait); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/ShutdownIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ShutdownIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ShutdownIT.java index 8d58821..d194367 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/ShutdownIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ShutdownIT.java @@ -19,11 +19,13 @@ package org.apache.accumulo.test.functional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.test.TestIngest; import org.apache.accumulo.test.TestRandomDeletes; @@ -90,7 +92,10 @@ public class ShutdownIT extends ConfigurableMacIT { @Test(timeout = 2 * 60 * 1000) public void adminStop() throws Exception { - Connector c = getConnector(); + runAdminStopTest(getConnector(), cluster); + } + + static void runAdminStopTest(Connector c, MiniAccumuloCluster cluster) throws InterruptedException, IOException { assertEquals(0, cluster.exec(TestIngest.class, "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable").waitFor()); List tabletServers = c.instanceOperations().getTabletServers(); assertEquals(2, tabletServers.size()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/SimpleMacIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SimpleMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SimpleMacIT.java index 9086f13..10db515 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/SimpleMacIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/SimpleMacIT.java @@ -26,6 +26,8 @@ import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.minicluster.MiniAccumuloConfig; import org.apache.accumulo.minicluster.MiniAccumuloInstance; import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.AfterClass; import org.junit.BeforeClass; public class SimpleMacIT extends AbstractMacIT { @@ -39,6 +41,7 @@ public class SimpleMacIT extends AbstractMacIT { if (getInstanceOneConnector() == null && cluster == null) { folder = createSharedTestDir(SimpleMacIT.class.getName()); MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder, ROOT_PASSWORD); + configureForEnvironment(cfg, SimpleMacIT.class, createSharedTestDir(SimpleMacIT.class.getName() + "-ssl")); cluster = new MiniAccumuloCluster(cfg); cluster.start(); Runtime.getRuntime().addShutdownHook(new Thread() { @@ -61,6 +64,20 @@ public class SimpleMacIT extends AbstractMacIT { return (getInstanceOneConnector() == null ? cluster.getConfig().getDir() : getInstanceOnePath()).getAbsolutePath(); } + public static MiniAccumuloCluster getStaticCluster() { + return cluster; + } + + public static File getFolder() { + return folder; + } + + @After + public void cleanUp() throws Exception {} + + @AfterClass + public static void tearDown() throws Exception {} + private static Connector getInstanceOneConnector() { try { return new MiniAccumuloInstance("instance1", getInstanceOnePath()).getConnector("root", new PasswordToken(ROOT_PASSWORD)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/SslIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SslIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SslIT.java new file mode 100644 index 0000000..6a29ad7 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/functional/SslIT.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.functional; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.minicluster.MiniAccumuloConfig; +import org.junit.Test; + +/** + * Do a selection of ITs with SSL turned on that cover a range of different connection scenarios. Note that you can run *all* the ITs against SSL-enabled mini + * clusters with `mvn verify -DuseSslForIT` + * + */ +public class SslIT extends ConfigurableMacIT { + @Override + public void configure(MiniAccumuloConfig cfg) { + super.configure(cfg); + configureForSsl(cfg, createSharedTestDir(this.getClass().getName() + "-ssl")); + } + + @Test(timeout = 60 * 1000) + public void binary() throws AccumuloException, AccumuloSecurityException, Exception { + getConnector().tableOperations().create("bt"); + BinaryIT.runTest(getConnector()); + } + + @Test(timeout = 2 * 60 * 1000) + public void concurrency() throws Exception { + ConcurrencyIT.runTest(getConnector()); + } + + @Test(timeout = 2 * 60 * 1000) + public void adminStop() throws Exception { + ShutdownIT.runAdminStopTest(getConnector(), getCluster()); + } + + @Test(timeout = 2 * 60 * 1000) + public void bulk() throws Exception { + BulkIT.runTest(getConnector(), getTableNames(1)[0]); + } + + @Test(timeout = 60 * 1000) + public void mapReduce() throws Exception { + MapReduceIT.runTest(getConnector(), getCluster()); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/functional/SslWithClientAuthIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SslWithClientAuthIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SslWithClientAuthIT.java new file mode 100644 index 0000000..c40e2b3 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/functional/SslWithClientAuthIT.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.functional; + +import java.util.Map; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.minicluster.MiniAccumuloConfig; +import org.junit.Test; + +/** + * Run all the same tests as SslIT, but with client auth turned on. + * + * All the methods are overridden just to make it easier to run individual tests from an IDE. + * + */ +public class SslWithClientAuthIT extends SslIT { + @Override + public void configure(MiniAccumuloConfig cfg) { + super.configure(cfg); + Map site = cfg.getSiteConfig(); + site.put(Property.INSTANCE_RPC_SSL_CLIENT_AUTH.getKey(), "true"); + cfg.setSiteConfig(site); + } + + @Override + @Test(timeout = 60000) + public void binary() throws AccumuloException, AccumuloSecurityException, Exception { + super.binary(); + } + + @Override + @Test(timeout = 120000) + public void concurrency() throws Exception { + super.concurrency(); + } + + @Override + @Test(timeout = 120000) + public void adminStop() throws Exception { + super.adminStop(); + } + + @Override + @Test(timeout = 120000) + public void bulk() throws Exception { + super.bulk(); + } + + @Override + @Test(timeout = 60000) + public void mapReduce() throws Exception { + super.mapReduce(); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/util/CertUtils.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/util/CertUtils.java b/test/src/test/java/org/apache/accumulo/test/util/CertUtils.java new file mode 100644 index 0000000..bb7b16d --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/util/CertUtils.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Security; +import java.security.UnrecoverableKeyException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.util.Calendar; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.accumulo.core.cli.Help; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.commons.io.FileExistsException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.Logger; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x500.style.IETFUtils; +import org.bouncycastle.asn1.x500.style.RFC4519Style; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.asn1.x509.X509Extension; +import org.bouncycastle.cert.CertIOException; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jce.provider.X509CertificateObject; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; + +public class CertUtils { + private static final Logger log = Logger.getLogger(CertUtils.class); + static { + Security.addProvider(new BouncyCastleProvider()); + } + + static class Opts extends Help { + @Parameter(description = "generate-all | generate-local | generate-self-trusted", required = true, arity = 1) + List operation = null; + + @Parameter(names = {"--local-keystore"}, description = "Target path for generated keystore") + String localKeystore = null; + + @Parameter(names = {"--root-keystore"}, description = "Path to root truststore, generated with generate-all, or used for signing with generate-local") + String rootKeystore = null; + + @Parameter(names = {"--root-truststore"}, description = "Target path for generated public root truststore") + String truststore = null; + + @Parameter(names = {"--keystore-type"}, description = "Type of keystore file to use") + String keystoreType = "JKS"; + + @Parameter( + names = {"--keystore-password"}, + description = "Password used to encrypt keystores. If omitted, the instance-wide secret will be used. If specified, the password must also be explicitly configured in Accumulo.") + String keystorePassword = null; + + @Parameter(names = {"--key-name-prefix"}, description = "Prefix for names of generated keys") + String keyNamePrefix = CertUtils.class.getSimpleName(); + + @Parameter(names = {"--issuer-rdn"}, description = "RDN string for issuer, for example: 'c=US,o=My Organization,cn=My Name'") + String issuerDirString = "o=Apache Accumulo"; + + @Parameter(names = "--site-file", description = "Load configuration from the given site file") + public String siteFile = null; + + @Parameter(names = "--signing-algorithm", description = "Algorithm used to sign certificates") + public String signingAlg = "SHA256WITHRSA"; + + @Parameter(names = "--encryption-algorithm", description = "Algorithm used to encrypt private keys") + public String encryptionAlg = "RSA"; + + @Parameter(names = "--keysize", description = "Key size used by encryption algorithm") + public int keysize = 2048; + + @SuppressWarnings("deprecation") + public AccumuloConfiguration getConfiguration() { + if (siteFile == null) { + return AccumuloConfiguration.getSiteConfiguration(); + } else { + return new AccumuloConfiguration() { + Configuration xml = new Configuration(); + { + xml.addResource(new Path(siteFile)); + } + + @Override + public Iterator> iterator() { + TreeMap map = new TreeMap(); + for (Entry props : DefaultConfiguration.getInstance()) + map.put(props.getKey(), props.getValue()); + for (Entry props : xml) + map.put(props.getKey(), props.getValue()); + return map.entrySet().iterator(); + } + + @Override + public String get(Property property) { + String value = xml.get(property.getKey()); + if (value != null) + return value; + return DefaultConfiguration.getInstance().get(property); + } + + @Override + public void getProperties(Map props, PropertyFilter filter) { + for (Entry entry : this) + if (filter.accept(entry.getKey())) + props.put(entry.getKey(), entry.getValue()); + } + }; + } + } + } + + public static void main(String[] args) throws Exception { + Opts opts = new Opts(); + opts.parseArgs(CertUtils.class.getName(), args); + String operation = opts.operation.get(0); + + String keyPassword = opts.keystorePassword; + if (keyPassword == null) + keyPassword = getDefaultKeyPassword(); + CertUtils certUtils = new CertUtils(opts.keystoreType, opts.issuerDirString, opts.encryptionAlg, opts.keysize, opts.signingAlg); + + if ("generate-all".equals(operation)) { + certUtils.createAll(new File(opts.rootKeystore), new File(opts.localKeystore), new File(opts.truststore), opts.keyNamePrefix, keyPassword); + } else if ("generate-local".equals(operation)) { + certUtils.createSignedCert(new File(opts.localKeystore), opts.keyNamePrefix + "-local", "", opts.rootKeystore, ""); + } else if ("generate-self-trusted".equals(operation)) { + certUtils.createSelfSignedCert(new File(opts.truststore), opts.keyNamePrefix + "-selfTrusted", ""); + } else { + JCommander jcommander = new JCommander(opts); + jcommander.setProgramName(CertUtils.class.getName()); + jcommander.usage(); + System.err.println("Unrecognized operation: " + opts.operation); + System.exit(0); + } + } + + @SuppressWarnings("deprecation") + private static String getDefaultKeyPassword() { + return AccumuloConfiguration.getSiteConfiguration().get(Property.INSTANCE_SECRET); + } + + private String issuerDirString; + private String keystoreType; + private String encryptionAlgorithm; + private int keysize; + private String signingAlgorithm; + + public CertUtils(String keystoreType, String issuerDirString, String encryptionAlgorithm, int keysize, String signingAlgorithm) { + super(); + this.keystoreType = keystoreType; + this.issuerDirString = issuerDirString; + this.encryptionAlgorithm = encryptionAlgorithm; + this.keysize = keysize; + this.signingAlgorithm = signingAlgorithm; + } + + public void createAll(File rootKeystoreFile, File localKeystoreFile, File trustStoreFile, String keyNamePrefix, String systemPassword) + throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException, OperatorCreationException, AccumuloSecurityException, + NoSuchProviderException, UnrecoverableKeyException, FileNotFoundException { + createSelfSignedCert(rootKeystoreFile, keyNamePrefix + "-root", systemPassword); + createSignedCert(localKeystoreFile, keyNamePrefix + "-local", systemPassword, rootKeystoreFile.getAbsolutePath(), systemPassword); + createPublicCert(trustStoreFile, keyNamePrefix + "-public", rootKeystoreFile.getAbsolutePath(), systemPassword); + } + + public void createPublicCert(File targetKeystoreFile, String keyName, String rootKeystorePath, String rootKeystorePassword) throws NoSuchAlgorithmException, + CertificateException, FileNotFoundException, IOException, KeyStoreException, UnrecoverableKeyException { + KeyStore signerKeystore = KeyStore.getInstance(keystoreType); + char[] signerPasswordArray = rootKeystorePassword.toCharArray(); + signerKeystore.load(new FileInputStream(rootKeystorePath), signerPasswordArray); + Certificate rootCert = findCert(signerKeystore); + + KeyStore keystore = KeyStore.getInstance(keystoreType); + keystore.load(null, null); + keystore.setCertificateEntry(keyName + "Cert", rootCert); + keystore.store(new FileOutputStream(targetKeystoreFile), new char[0]); + } + + public void createSignedCert(File targetKeystoreFile, String keyName, String keystorePassword, String signerKeystorePath, String signerKeystorePassword) + throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException, OperatorCreationException, AccumuloSecurityException, + UnrecoverableKeyException, NoSuchProviderException { + KeyStore signerKeystore = KeyStore.getInstance(keystoreType); + char[] signerPasswordArray = signerKeystorePassword.toCharArray(); + signerKeystore.load(new FileInputStream(signerKeystorePath), signerPasswordArray); + Certificate signerCert = findCert(signerKeystore); + PrivateKey signerKey = findPrivateKey(signerKeystore, signerPasswordArray); + + KeyPair kp = generateKeyPair(); + X509CertificateObject cert = generateCert(keyName, kp, false, signerCert.getPublicKey(), signerKey); + + char[] password = keystorePassword.toCharArray(); + KeyStore keystore = KeyStore.getInstance(keystoreType); + keystore.load(null, null); + keystore.setCertificateEntry(keyName + "Cert", cert); + keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new Certificate[] {cert, signerCert}); + keystore.store(new FileOutputStream(targetKeystoreFile), password); + } + + public void createSelfSignedCert(File targetKeystoreFile, String keyName, String keystorePassword) throws KeyStoreException, CertificateException, + NoSuchAlgorithmException, IOException, OperatorCreationException, AccumuloSecurityException, NoSuchProviderException { + if (targetKeystoreFile.exists()) { + throw new FileExistsException(targetKeystoreFile); + } + + KeyPair kp = generateKeyPair(); + + X509CertificateObject cert = generateCert(keyName, kp, true, kp.getPublic(), kp.getPrivate()); + + char[] password = keystorePassword.toCharArray(); + KeyStore keystore = KeyStore.getInstance(keystoreType); + keystore.load(null, null); + keystore.setCertificateEntry(keyName + "Cert", cert); + keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new Certificate[] {cert}); + keystore.store(new FileOutputStream(targetKeystoreFile), password); + } + + private KeyPair generateKeyPair() throws NoSuchAlgorithmException, NoSuchProviderException { + KeyPairGenerator gen = KeyPairGenerator.getInstance(encryptionAlgorithm); + gen.initialize(keysize); + return gen.generateKeyPair(); + } + + private X509CertificateObject generateCert(String keyName, KeyPair kp, boolean isCertAuthority, PublicKey signerPublicKey, PrivateKey signerPrivateKey) + throws IOException, CertIOException, OperatorCreationException, CertificateException, NoSuchAlgorithmException { + Calendar startDate = Calendar.getInstance(); + Calendar endDate = Calendar.getInstance(); + endDate.add(Calendar.YEAR, 100); + + BigInteger serialNumber = BigInteger.valueOf((startDate.getTimeInMillis())); + X500Name issuer = new X500Name(IETFUtils.rDNsFromString(issuerDirString, RFC4519Style.INSTANCE)); + JcaX509v3CertificateBuilder certGen = new JcaX509v3CertificateBuilder(issuer, serialNumber, startDate.getTime(), endDate.getTime(), issuer, kp.getPublic()); + JcaX509ExtensionUtils extensionUtils = new JcaX509ExtensionUtils(); + certGen.addExtension(X509Extension.subjectKeyIdentifier, false, extensionUtils.createSubjectKeyIdentifier(kp.getPublic())); + certGen.addExtension(X509Extension.basicConstraints, false, new BasicConstraints(isCertAuthority)); + certGen.addExtension(X509Extension.authorityKeyIdentifier, false, extensionUtils.createAuthorityKeyIdentifier(signerPublicKey)); + if (isCertAuthority) { + certGen.addExtension(X509Extension.keyUsage, true, new KeyUsage(KeyUsage.keyCertSign)); + } + X509CertificateHolder cert = certGen.build(new JcaContentSignerBuilder(signingAlgorithm).build(signerPrivateKey)); + return new X509CertificateObject(cert.toASN1Structure()); + } + + static Certificate findCert(KeyStore keyStore) throws KeyStoreException { + Enumeration aliases = keyStore.aliases(); + Certificate cert = null; + while (aliases.hasMoreElements()) { + String alias = aliases.nextElement(); + if (keyStore.isCertificateEntry(alias)) { + if (cert == null) { + cert = keyStore.getCertificate(alias); + } else { + log.warn("Found multiple certificates in keystore. Ignoring " + alias); + } + } + } + if (cert == null) { + throw new KeyStoreException("Could not find cert in keystore"); + } + return cert; + } + + static PrivateKey findPrivateKey(KeyStore keyStore, char[] keystorePassword) throws UnrecoverableKeyException, KeyStoreException, NoSuchAlgorithmException { + Enumeration aliases = keyStore.aliases(); + PrivateKey key = null; + while (aliases.hasMoreElements()) { + String alias = aliases.nextElement(); + if (keyStore.isKeyEntry(alias)) { + if (key == null) { + key = (PrivateKey) keyStore.getKey(alias, keystorePassword); + } else { + log.warn("Found multiple keys in keystore. Ignoring " + alias); + } + } + } + if (key == null) { + throw new KeyStoreException("Could not find private key in keystore"); + } + return key; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/7038755b/test/src/test/java/org/apache/accumulo/test/util/CertUtilsTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/util/CertUtilsTest.java b/test/src/test/java/org/apache/accumulo/test/util/CertUtilsTest.java new file mode 100644 index 0000000..bb2a933 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/util/CertUtilsTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.util; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileInputStream; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.SignatureException; +import java.security.cert.Certificate; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class CertUtilsTest { + private static final String KEYSTORE_TYPE = "JKS"; + private static final String PASSWORD = "CertUtilsTestPassword"; + private static final char[] PASSWORD_CHARS = PASSWORD.toCharArray(); + private static final String RDN_STRING = "o=Apache Accumulo,cn=CertUtilsTest"; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private CertUtils getUtils() { + return new CertUtils(KEYSTORE_TYPE, RDN_STRING, "RSA", 2048, "sha1WithRSAEncryption"); + } + + @Test + public void createSelfSigned() throws Exception { + CertUtils certUtils = getUtils(); + File keyStoreFile = new File(folder.getRoot(), "selfsigned.jks"); + certUtils.createSelfSignedCert(keyStoreFile, "test", PASSWORD); + + KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE); + keyStore.load(new FileInputStream(keyStoreFile), PASSWORD_CHARS); + Certificate cert = CertUtils.findCert(keyStore); + + cert.verify(cert.getPublicKey()); // throws exception if it can't be verified + } + + @Test + public void createPublicSelfSigned() throws Exception { + CertUtils certUtils = getUtils(); + File rootKeyStoreFile = new File(folder.getRoot(), "root.jks"); + certUtils.createSelfSignedCert(rootKeyStoreFile, "test", PASSWORD); + File publicKeyStoreFile = new File(folder.getRoot(), "public.jks"); + certUtils.createPublicCert(publicKeyStoreFile, "test", rootKeyStoreFile.getAbsolutePath(), PASSWORD); + + KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE); + keyStore.load(new FileInputStream(publicKeyStoreFile), new char[0]); + try { + CertUtils.findPrivateKey(keyStore, PASSWORD_CHARS); + fail("expected not to find private key in keystore"); + } catch (KeyStoreException e) { + assertTrue(e.getMessage().contains("private key")); + } + Certificate cert = CertUtils.findCert(keyStore); + cert.verify(cert.getPublicKey()); // throws exception if it can't be verified + } + + @Test + public void createSigned() throws Exception { + CertUtils certUtils = getUtils(); + File rootKeyStoreFile = new File(folder.getRoot(), "root.jks"); + certUtils.createSelfSignedCert(rootKeyStoreFile, "test", PASSWORD); + File signedKeyStoreFile = new File(folder.getRoot(), "signed.jks"); + certUtils.createSignedCert(signedKeyStoreFile, "test", PASSWORD, rootKeyStoreFile.getAbsolutePath(), PASSWORD); + + KeyStore rootKeyStore = KeyStore.getInstance(KEYSTORE_TYPE); + rootKeyStore.load(new FileInputStream(rootKeyStoreFile), PASSWORD_CHARS); + Certificate rootCert = CertUtils.findCert(rootKeyStore); + + KeyStore signedKeyStore = KeyStore.getInstance(KEYSTORE_TYPE); + signedKeyStore.load(new FileInputStream(signedKeyStoreFile), PASSWORD_CHARS); + Certificate signedCert = CertUtils.findCert(signedKeyStore); + + try { + signedCert.verify(signedCert.getPublicKey()); + fail("signed cert should not be able to verify itself"); + } catch (SignatureException e) { + // expected + } + + signedCert.verify(rootCert.getPublicKey()); // throws exception if it can't be verified + } + + @Test + public void publicOnlyVerfication() throws Exception { + // this approximates the real life scenario. the client will only have the public key of each + // cert (the root made by us as below, but the signed cert extracted by the SSL transport) + CertUtils certUtils = getUtils(); + File rootKeyStoreFile = new File(folder.getRoot(), "root.jks"); + certUtils.createSelfSignedCert(rootKeyStoreFile, "test", PASSWORD); + File publicRootKeyStoreFile = new File(folder.getRoot(), "publicroot.jks"); + certUtils.createPublicCert(publicRootKeyStoreFile, "test", rootKeyStoreFile.getAbsolutePath(), PASSWORD); + File signedKeyStoreFile = new File(folder.getRoot(), "signed.jks"); + certUtils.createSignedCert(signedKeyStoreFile, "test", PASSWORD, rootKeyStoreFile.getAbsolutePath(), PASSWORD); + File publicSignedKeyStoreFile = new File(folder.getRoot(), "publicsigned.jks"); + certUtils.createPublicCert(publicSignedKeyStoreFile, "test", signedKeyStoreFile.getAbsolutePath(), PASSWORD); + + KeyStore rootKeyStore = KeyStore.getInstance(KEYSTORE_TYPE); + rootKeyStore.load(new FileInputStream(publicRootKeyStoreFile), new char[0]); + KeyStore signedKeyStore = KeyStore.getInstance(KEYSTORE_TYPE); + signedKeyStore.load(new FileInputStream(publicSignedKeyStoreFile), new char[0]); + Certificate rootCert = CertUtils.findCert(rootKeyStore); + Certificate signedCert = CertUtils.findCert(signedKeyStore); + + try { + signedCert.verify(signedCert.getPublicKey()); + fail("signed cert should not be able to verify itself"); + } catch (SignatureException e) { + // expected + } + + signedCert.verify(rootCert.getPublicKey()); // throws exception if it can't be verified + } + + @Test + public void signingChain() throws Exception { + // no reason the keypair we generate for the tservers need to be able to sign anything, + // but this is a way to make sure the private and public keys created actually correspond. + CertUtils certUtils = getUtils(); + File rootKeyStoreFile = new File(folder.getRoot(), "root.jks"); + certUtils.createSelfSignedCert(rootKeyStoreFile, "test", PASSWORD); + File signedCaKeyStoreFile = new File(folder.getRoot(), "signedca.jks"); + certUtils.createSignedCert(signedCaKeyStoreFile, "test", PASSWORD, rootKeyStoreFile.getAbsolutePath(), PASSWORD); + File signedLeafKeyStoreFile = new File(folder.getRoot(), "signedleaf.jks"); + certUtils.createSignedCert(signedLeafKeyStoreFile, "test", PASSWORD, signedCaKeyStoreFile.getAbsolutePath(), PASSWORD); + + KeyStore caKeyStore = KeyStore.getInstance(KEYSTORE_TYPE); + caKeyStore.load(new FileInputStream(signedCaKeyStoreFile), PASSWORD_CHARS); + Certificate caCert = CertUtils.findCert(caKeyStore); + + KeyStore leafKeyStore = KeyStore.getInstance(KEYSTORE_TYPE); + leafKeyStore.load(new FileInputStream(signedLeafKeyStoreFile), PASSWORD_CHARS); + Certificate leafCert = CertUtils.findCert(leafKeyStore); + + leafCert.verify(caCert.getPublicKey()); // throws exception if it can't be verified + } +}