From commits-return-22524-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Wed Jan 23 16:15:19 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 8189A180771 for ; Wed, 23 Jan 2019 16:15:17 +0100 (CET) Received: (qmail 32486 invoked by uid 500); 23 Jan 2019 15:15:16 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 32477 invoked by uid 99); 23 Jan 2019 15:15:16 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 23 Jan 2019 15:15:16 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id E64C9854E5; Wed, 23 Jan 2019 15:15:15 +0000 (UTC) Date: Wed, 23 Jan 2019 15:15:15 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch master updated: fixes #908 improve and fix reading stdout from test (#911) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <154825651551.26366.6921506335333641376@gitbox.apache.org> From: kturner@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 7db5a0d8b4ca9fb3abde81c4f1caa6ff24af5685 X-Git-Newrev: 1bb6226aa9c10060d7e01c66dbc5145c269aaecf X-Git-Rev: 1bb6226aa9c10060d7e01c66dbc5145c269aaecf X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/master by this push: new 1bb6226 fixes #908 improve and fix reading stdout from test (#911) 1bb6226 is described below commit 1bb6226aa9c10060d7e01c66dbc5145c269aaecf Author: Keith Turner AuthorDate: Wed Jan 23 10:09:17 2019 -0500 fixes #908 improve and fix reading stdout from test (#911) --- .../accumulo/hadoop/its/mapreduce/RowHashIT.java | 2 +- .../MiniAccumuloClusterControl.java | 40 ++++++---------- .../miniclusterImpl/MiniAccumuloClusterImpl.java | 53 ++++++++++++++++------ .../apache/accumulo/test/ConditionalWriterIT.java | 2 +- .../accumulo/test/RewriteTabletDirectoriesIT.java | 7 +-- .../org/apache/accumulo/test/ShellServerIT.java | 2 +- .../ThriftServerBindsBeforeZooKeeperLockIT.java | 5 +- .../test/TracerRecoversAfterOfflineTableIT.java | 2 +- .../accumulo/test/VerifySerialRecoveryIT.java | 10 ++-- .../java/org/apache/accumulo/test/VolumeIT.java | 11 +++-- .../test/functional/ConfigurableMacBase.java | 2 +- .../test/functional/FunctionalTestUtils.java | 23 ++-------- .../test/functional/GarbageCollectorIT.java | 22 ++++----- .../test/functional/HalfDeadTServerIT.java | 2 +- .../accumulo/test/functional/KerberosProxyIT.java | 2 +- .../test/functional/LateLastContactIT.java | 2 +- .../test/functional/MetadataMaxFilesIT.java | 2 +- .../test/functional/RecoveryWithEmptyRFileIT.java | 2 +- .../accumulo/test/functional/ShutdownIT.java | 33 ++++++++------ .../accumulo/test/functional/WALSunnyDayIT.java | 4 +- .../accumulo/test/mapreduce/MapReduceIT.java | 8 ++-- .../accumulo/test/replication/ReplicationIT.java | 2 +- 22 files changed, 121 insertions(+), 117 deletions(-) diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/RowHashIT.java b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/RowHashIT.java index f6ad850..414a4c0 100644 --- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/RowHashIT.java +++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/RowHashIT.java @@ -97,7 +97,7 @@ public class RowHashIT extends ConfigurableMacBase { ClientInfo info = ClientInfo.from(c.properties()); Process hash = cluster.exec(RowHash.class, Collections.singletonList(hadoopTmpDirArg), "-i", info.getInstanceName(), "-z", info.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "-t", - tablename, "--column", input_cfcq); + tablename, "--column", input_cfcq).getProcess(); assertEquals(0, hash.waitFor()); try (Scanner s = c.createScanner(tablename, Authorizations.EMPTY)) { diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java index 96eb563..bcddd94 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java @@ -18,9 +18,7 @@ package org.apache.accumulo.miniclusterImpl; import static java.util.Objects.requireNonNull; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -34,6 +32,7 @@ import org.apache.accumulo.cluster.ClusterControl; import org.apache.accumulo.gc.SimpleGarbageCollector; import org.apache.accumulo.master.Master; import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.tracer.TraceServer; @@ -69,7 +68,7 @@ public class MiniAccumuloClusterControl implements ClusterControl { @Override public int exec(Class clz, String[] args) throws IOException { - Process p = cluster.exec(clz, args); + Process p = cluster.exec(clz, args).getProcess(); int exitCode; try { exitCode = p.waitFor(); @@ -85,35 +84,22 @@ public class MiniAccumuloClusterControl implements ClusterControl { justification = "code runs in same security context as user who provided input file name") @Override public Entry execWithStdout(Class clz, String[] args) throws IOException { - Process p = cluster.exec(clz, args); + ProcessInfo pi = cluster.exec(clz, args); int exitCode; try { - exitCode = p.waitFor(); + exitCode = pi.getProcess().waitFor(); } catch (InterruptedException e) { log.warn("Interrupted waiting for process to exit", e); Thread.currentThread().interrupt(); throw new IOException(e); } - return Maps.immutableEntry(exitCode, readAll(new FileInputStream(cluster.getConfig().getLogDir() - + "/" + clz.getSimpleName() + "_" + p.hashCode() + ".out"))); - } - - private String readAll(InputStream is) throws IOException { - byte[] buffer = new byte[4096]; - StringBuilder result = new StringBuilder(); - while (true) { - int n = is.read(buffer); - if (n <= 0) - break; - result.append(new String(buffer, 0, n)); - } - return result.toString(); + return Maps.immutableEntry(exitCode, pi.readStdOut()); } @Override public void adminStopAll() throws IOException { - Process p = cluster.exec(Admin.class, "stopAll"); + Process p = cluster.exec(Admin.class, "stopAll").getProcess(); try { p.waitFor(); } catch (InterruptedException e) { @@ -147,34 +133,36 @@ public class MiniAccumuloClusterControl implements ClusterControl { int count = 0; for (int i = tabletServerProcesses.size(); count < limit && i < cluster.getConfig().getNumTservers(); i++, ++count) { - tabletServerProcesses.add(cluster._exec(TabletServer.class, server, configOverrides)); + tabletServerProcesses + .add(cluster._exec(TabletServer.class, server, configOverrides).getProcess()); } } break; case MASTER: if (masterProcess == null) { - masterProcess = cluster._exec(Master.class, server, configOverrides); + masterProcess = cluster._exec(Master.class, server, configOverrides).getProcess(); } break; case ZOOKEEPER: if (zooKeeperProcess == null) { zooKeeperProcess = cluster._exec(ZooKeeperServerMain.class, server, configOverrides, - cluster.getZooCfgFile().getAbsolutePath()); + cluster.getZooCfgFile().getAbsolutePath()).getProcess(); } break; case GARBAGE_COLLECTOR: if (gcProcess == null) { - gcProcess = cluster._exec(SimpleGarbageCollector.class, server, configOverrides); + gcProcess = cluster._exec(SimpleGarbageCollector.class, server, configOverrides) + .getProcess(); } break; case MONITOR: if (monitor == null) { - monitor = cluster._exec(Monitor.class, server, configOverrides); + monitor = cluster._exec(Monitor.class, server, configOverrides).getProcess(); } break; case TRACER: if (tracer == null) { - tracer = cluster._exec(TraceServer.class, server, configOverrides); + tracer = cluster._exec(TraceServer.class, server, configOverrides).getProcess(); } break; default: diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java index 2c7d21e..ac4e703 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java @@ -20,14 +20,18 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly; import java.io.File; +import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.InetSocketAddress; import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; @@ -86,6 +90,7 @@ import org.apache.accumulo.server.util.PortUtils; import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory; import org.apache.accumulo.start.Main; import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil; +import org.apache.commons.io.IOUtils; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.impl.VFSClassLoader; import org.apache.hadoop.conf.Configuration; @@ -140,11 +145,11 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { return zooCfgFile; } - public Process exec(Class clazz, String... args) throws IOException { + public ProcessInfo exec(Class clazz, String... args) throws IOException { return exec(clazz, null, args); } - public Process exec(Class clazz, List jvmArgs, String... args) throws IOException { + public ProcessInfo exec(Class clazz, List jvmArgs, String... args) throws IOException { ArrayList jvmArgs2 = new ArrayList<>(1 + (jvmArgs == null ? 0 : jvmArgs.size())); jvmArgs2.add("-Xmx" + config.getDefaultMemory()); if (jvmArgs != null) @@ -227,9 +232,32 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { } } + public static class ProcessInfo { + + private final Process process; + private final File stdOut; + + public ProcessInfo(Process process, File stdOut) { + this.process = process; + this.stdOut = stdOut; + } + + public Process getProcess() { + return process; + } + + public String readStdOut() { + try (InputStream in = new FileInputStream(stdOut)) { + return IOUtils.toString(in, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + @SuppressFBWarnings(value = "COMMAND_INJECTION", justification = "mini runs in the same security context as user providing the args") - private Process _exec(Class clazz, List extraJvmOpts, String... args) + private ProcessInfo _exec(Class clazz, List extraJvmOpts, String... args) throws IOException { String javaHome = System.getProperty("java.home"); String javaBin = javaHome + File.separator + "bin" + File.separator + "java"; @@ -282,21 +310,18 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { Integer hashcode = builder.hashCode(); - builder = builder - .redirectError( - new File(config.getLogDir(), clazz.getSimpleName() + "_" + hashcode + ".err")) - .redirectOutput( - new File(config.getLogDir(), clazz.getSimpleName() + "_" + hashcode + ".out")); + File stdOut = new File(config.getLogDir(), clazz.getSimpleName() + "_" + hashcode + ".out"); + File stdErr = new File(config.getLogDir(), clazz.getSimpleName() + "_" + hashcode + ".err"); - Process process = builder.start(); + Process process = builder.redirectError(stdErr).redirectOutput(stdOut).start(); cleanup.add(process); - return process; + return new ProcessInfo(process, stdOut); } - public Process _exec(Class clazz, ServerType serverType, Map configOverrides, - String... args) throws IOException { + public ProcessInfo _exec(Class clazz, ServerType serverType, + Map configOverrides, String... args) throws IOException { List jvmOpts = new ArrayList<>(); jvmOpts.add("-Xmx" + config.getMemory(serverType)); if (configOverrides != null && !configOverrides.isEmpty()) { @@ -576,7 +601,7 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { args.add(config.getRootPassword()); } - Process initProcess = exec(Initialize.class, args.toArray(new String[0])); + Process initProcess = exec(Initialize.class, args.toArray(new String[0])).getProcess(); int ret = initProcess.waitFor(); if (ret != 0) { throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " @@ -594,7 +619,7 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { int ret = 0; for (int i = 0; i < 5; i++) { ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()) - .waitFor(); + .getProcess().waitFor(); if (ret == 0) break; sleepUninterruptibly(1, TimeUnit.SECONDS); diff --git a/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java index 0bc2b95..b4b4940 100644 --- a/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java @@ -1537,7 +1537,7 @@ public class ConditionalWriterIT extends AccumuloClusterHarness { AccumuloCluster cluster = getCluster(); MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster; if (!client.tableOperations().exists("trace")) { - tracer = mac.exec(TraceServer.class); + tracer = mac.exec(TraceServer.class).getProcess(); while (!client.tableOperations().exists("trace")) { sleepUninterruptibly(1, TimeUnit.SECONDS); } diff --git a/test/src/main/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java b/test/src/main/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java index d35a6c7..ebf12f8 100644 --- a/test/src/main/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java @@ -125,7 +125,7 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacBase { // This should fail: only one volume assertEquals(1, cluster.exec(RandomizeVolumes.class, "-z", cluster.getZooKeepers(), "-i", - getClientInfo().getInstanceName(), "-t", tableName).waitFor()); + getClientInfo().getInstanceName(), "-t", tableName).getProcess().waitFor()); cluster.stop(); @@ -136,11 +136,12 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacBase { conf.save(cluster.getAccumuloPropertiesPath()); // initialize volume - assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor()); + assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").getProcess().waitFor()); cluster.start(); // change the directory entries - assertEquals(0, cluster.exec(Admin.class, "randomizeVolumes", "-t", tableName).waitFor()); + assertEquals(0, + cluster.exec(Admin.class, "randomizeVolumes", "-t", tableName).getProcess().waitFor()); // verify a more equal sharing int v1Count = 0, v2Count = 0; diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java index 2b6e30e..24ac840 100644 --- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java @@ -302,7 +302,7 @@ public class ShellServerIT extends SharedMiniClusterBase { System.setProperty("HOME", rootPath); System.setProperty("hadoop.tmp.dir", userDir + "/target/hadoop-tmp"); - traceProcess = getCluster().exec(TraceServer.class); + traceProcess = getCluster().exec(TraceServer.class).getProcess(); AccumuloClient client = getCluster().createAccumuloClient(getPrincipal(), getToken()); TableOperations tops = client.tableOperations(); diff --git a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java index 1e59ccb..ee5a9a8 100644 --- a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java @@ -284,7 +284,8 @@ public class ThriftServerBindsBeforeZooKeeperLockIT extends AccumuloClusterHarne throw new IllegalArgumentException("Irrelevant server type for test"); } - return cluster._exec(service, serverType, - ImmutableMap.of(property.getKey(), Integer.toString(port))); + return cluster + ._exec(service, serverType, ImmutableMap.of(property.getKey(), Integer.toString(port))) + .getProcess(); } } diff --git a/test/src/main/java/org/apache/accumulo/test/TracerRecoversAfterOfflineTableIT.java b/test/src/main/java/org/apache/accumulo/test/TracerRecoversAfterOfflineTableIT.java index eb715ad..8e53a7b 100644 --- a/test/src/main/java/org/apache/accumulo/test/TracerRecoversAfterOfflineTableIT.java +++ b/test/src/main/java/org/apache/accumulo/test/TracerRecoversAfterOfflineTableIT.java @@ -58,7 +58,7 @@ public class TracerRecoversAfterOfflineTableIT extends ConfigurableMacBase { try (AccumuloClient client = createClient()) { if (!client.tableOperations().exists("trace")) { MiniAccumuloClusterImpl mac = cluster; - tracer = mac.exec(TraceServer.class); + tracer = mac.exec(TraceServer.class).getProcess(); while (!client.tableOperations().exists("trace")) { sleepUninterruptibly(1, TimeUnit.SECONDS); } diff --git a/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java index ef3d6b3..bd6d617 100644 --- a/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java @@ -31,11 +31,11 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.miniclusterImpl.ProcessReference; import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.test.functional.ConfigurableMacBase; -import org.apache.accumulo.test.functional.FunctionalTestUtils; import org.apache.accumulo.tserver.TabletServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.RawLocalFileSystem; @@ -92,13 +92,13 @@ public class VerifySerialRecoveryIT extends ConfigurableMacBase { // kill the tserver for (ProcessReference ref : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) getCluster().killProcess(ServerType.TABLET_SERVER, ref); - final Process ts = cluster.exec(TabletServer.class); + final ProcessInfo ts = cluster.exec(TabletServer.class); // wait for recovery Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator()); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); - ts.waitFor(); - String result = FunctionalTestUtils.readAll(cluster, TabletServer.class, ts); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + ts.getProcess().waitFor(); + String result = ts.readStdOut(); for (String line : result.split("\n")) { System.out.println(line); } diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java index 4347d82..5b2bb94 100644 --- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java @@ -280,6 +280,7 @@ public class VolumeIT extends ConfigurableMacBase { public void testAddVolumes() throws Exception { try (AccumuloClient client = createClient()) { String[] tableNames = getUniqueNames(2); + PropertiesConfiguration conf = new PropertiesConfiguration(); String uuid = verifyAndShutdownCluster(client, conf, tableNames[0]); @@ -288,7 +289,7 @@ public class VolumeIT extends ConfigurableMacBase { conf.save(cluster.getAccumuloPropertiesPath()); // initialize volume - assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor()); + assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").getProcess().waitFor()); checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid); @@ -306,7 +307,7 @@ public class VolumeIT extends ConfigurableMacBase { verifyVolumesUsed(c, tableName, false, v1, v2); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); cluster.stop(); conf.load(cluster.getAccumuloPropertiesPath()); @@ -326,7 +327,7 @@ public class VolumeIT extends ConfigurableMacBase { conf.save(cluster.getAccumuloPropertiesPath()); // initialize volume - assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor()); + assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").getProcess().waitFor()); checkVolumesInitialized(Arrays.asList(v1, v2, v3), uuid); @@ -462,7 +463,7 @@ public class VolumeIT extends ConfigurableMacBase { verifyVolumesUsed(client, tableNames[0], false, v1, v2); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); cluster.stop(); PropertiesConfiguration conf = new PropertiesConfiguration(); @@ -505,7 +506,7 @@ public class VolumeIT extends ConfigurableMacBase { cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))); if (cleanShutdown) - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); cluster.stop(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ConfigurableMacBase.java b/test/src/main/java/org/apache/accumulo/test/functional/ConfigurableMacBase.java index aacd8b1..aa87f21 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ConfigurableMacBase.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ConfigurableMacBase.java @@ -214,6 +214,6 @@ public class ConfigurableMacBase extends AccumuloITBase { } protected Process exec(Class clazz, String... args) throws IOException { - return getCluster().exec(clazz, args); + return getCluster().exec(clazz, args).getProcess(); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java b/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java index 383388e..e8e0459 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java @@ -16,10 +16,10 @@ */ package org.apache.accumulo.test.functional; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Collection; @@ -52,9 +52,9 @@ import org.apache.accumulo.fate.AdminUtil; import org.apache.accumulo.fate.AdminUtil.FateStatus; import org.apache.accumulo.fate.ZooStore; import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; -import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory; import org.apache.accumulo.test.TestIngest; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -62,8 +62,6 @@ import org.apache.zookeeper.KeeperException; import com.google.common.collect.Iterators; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - public class FunctionalTestUtils { public static int countRFiles(AccumuloClient c, String tableName) throws Exception { @@ -154,22 +152,7 @@ public class FunctionalTestUtils { } public static String readAll(InputStream is) throws IOException { - byte[] buffer = new byte[4096]; - StringBuilder result = new StringBuilder(); - while (true) { - int n = is.read(buffer); - if (n <= 0) - break; - result.append(new String(buffer, 0, n)); - } - return result.toString(); - } - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by test") - public static String readAll(MiniAccumuloClusterImpl c, Class klass, Process p) - throws Exception { - return readAll(new FileInputStream( - c.getConfig().getLogDir() + "/" + klass.getSimpleName() + "_" + p.hashCode() + ".out")); + return IOUtils.toString(is, UTF_8); } static Mutation nm(String row, String cf, String cq, Value value) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java index 4aaf3f8..7c5f458 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java @@ -16,7 +16,6 @@ */ package org.apache.accumulo.test.functional; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -25,7 +24,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -56,6 +55,7 @@ import org.apache.accumulo.fate.zookeeper.ZooUtil; import org.apache.accumulo.gc.SimpleGarbageCollector; import org.apache.accumulo.minicluster.MemoryUnit; import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.miniclusterImpl.ProcessNotFoundException; import org.apache.accumulo.miniclusterImpl.ProcessReference; @@ -149,19 +149,17 @@ public class GarbageCollectorIT extends ConfigurableMacBase { try (AccumuloClient c = createClient()) { addEntries(c, new BatchWriterOpts()); cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE); - Process gc = cluster.exec(SimpleGarbageCollector.class); + ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class); sleepUninterruptibly(20, TimeUnit.SECONDS); String output = ""; while (!output.contains("delete candidates has exceeded")) { - byte buffer[] = new byte[10 * 1024]; try { - int n = gc.getInputStream().read(buffer); - output = new String(buffer, 0, n, UTF_8); - } catch (IOException ex) { + output = gc.readStdOut(); + } catch (UncheckedIOException ex) { break; } } - gc.destroy(); + gc.getProcess().destroy(); assertTrue(output.contains("delete candidates has exceeded")); } } @@ -222,19 +220,19 @@ public class GarbageCollectorIT extends ConfigurableMacBase { bw3.addMutation(createDelMutation("/", "", "", "")); bw3.close(); - Process gc = cluster.exec(SimpleGarbageCollector.class); + ProcessInfo gc = cluster.exec(SimpleGarbageCollector.class); try { String output = ""; while (!output.contains("Ignoring invalid deletion candidate")) { sleepUninterruptibly(250, TimeUnit.MILLISECONDS); try { - output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc); - } catch (IOException ioe) { + output = gc.readStdOut(); + } catch (UncheckedIOException ioe) { log.error("Could not read all from cluster.", ioe); } } } finally { - gc.destroy(); + gc.getProcess().destroy(); } try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java index 659e28e..b86378e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java @@ -158,7 +158,7 @@ public class HalfDeadTServerIT extends ConfigurableMacBase { assertEquals(1, c.instanceOperations().getTabletServers().size()); int rows = 100 * 1000; ingest = cluster.exec(TestIngest.class, "-u", "root", "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "-p", ROOT_PASSWORD, "--rows", rows + ""); + cluster.getZooKeepers(), "-p", ROOT_PASSWORD, "--rows", rows + "").getProcess(); sleepUninterruptibly(500, TimeUnit.MILLISECONDS); // block I/O with some side-channel trickiness diff --git a/test/src/main/java/org/apache/accumulo/test/functional/KerberosProxyIT.java b/test/src/main/java/org/apache/accumulo/test/functional/KerberosProxyIT.java index a7b5fdd..af7654c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/KerberosProxyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/KerberosProxyIT.java @@ -236,7 +236,7 @@ public class KerberosProxyIT extends AccumuloITBase { File proxyPropertiesFile = generateNewProxyConfiguration(cfg); File clientPropsFile = generateNewAccumuloClientConfiguration(cfg); return mac.exec(Proxy.class, "-p", proxyPropertiesFile.getCanonicalPath(), "-c", - clientPropsFile.getCanonicalPath()); + clientPropsFile.getCanonicalPath()).getProcess(); } /** diff --git a/test/src/main/java/org/apache/accumulo/test/functional/LateLastContactIT.java b/test/src/main/java/org/apache/accumulo/test/functional/LateLastContactIT.java index 1097625..dd5ecc1 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/LateLastContactIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/LateLastContactIT.java @@ -43,7 +43,7 @@ public class LateLastContactIT extends ConfigurableMacBase { @Test public void test() throws Exception { - Process zombie = cluster.exec(ZombieTServer.class); + Process zombie = cluster.exec(ZombieTServer.class).getProcess(); assertEquals(0, zombie.waitFor()); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java index b3fb553..fafaec8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java @@ -80,7 +80,7 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase { c.tableOperations().flush(RootTable.NAME, null, null, true); } log.info("shutting down"); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); cluster.stop(); log.info("starting up"); cluster.start(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RecoveryWithEmptyRFileIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RecoveryWithEmptyRFileIT.java index 09adeb3..4a88a90 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RecoveryWithEmptyRFileIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RecoveryWithEmptyRFileIT.java @@ -88,7 +88,7 @@ public class RecoveryWithEmptyRFileIT extends ConfigurableMacBase { Path rfile = new Path(entry.getKey().getColumnQualifier().toString()); log.debug("Removing rfile '{}'", rfile); cluster.getFileSystem().delete(rfile, false); - Process info = cluster.exec(CreateEmpty.class, rfile.toString()); + Process info = cluster.exec(CreateEmpty.class, rfile.toString()).getProcess(); assertEquals(0, info.waitFor()); } assertTrue(foundFile); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java index a87ce2e..9dab761 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java @@ -43,31 +43,35 @@ public class ShutdownIT extends ConfigurableMacBase { @Test public void shutdownDuringIngest() throws Exception { Process ingest = cluster.exec(TestIngest.class, "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable"); + cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable").getProcess(); sleepUninterruptibly(100, TimeUnit.MILLISECONDS); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); ingest.destroy(); } @Test public void shutdownDuringQuery() throws Exception { - assertEquals(0, cluster.exec(TestIngest.class, "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable").waitFor()); + assertEquals(0, + cluster.exec(TestIngest.class, "-i", cluster.getInstanceName(), "-z", + cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable") + .getProcess().waitFor()); Process verify = cluster.exec(VerifyIngest.class, "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD); + cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD).getProcess(); sleepUninterruptibly(100, TimeUnit.MILLISECONDS); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); verify.destroy(); } @Test public void shutdownDuringDelete() throws Exception { - assertEquals(0, cluster.exec(TestIngest.class, "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable").waitFor()); + assertEquals(0, + cluster.exec(TestIngest.class, "-i", cluster.getInstanceName(), "-z", + cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--createTable") + .getProcess().waitFor()); Process deleter = cluster.exec(TestRandomDeletes.class, "-i", cluster.getInstanceName(), "-z", - cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD); + cluster.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD).getProcess(); sleepUninterruptibly(100, TimeUnit.MILLISECONDS); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); deleter.destroy(); } @@ -91,7 +95,7 @@ public class ShutdownIT extends ConfigurableMacBase { }; async.start(); sleepUninterruptibly(100, TimeUnit.MILLISECONDS); - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); if (ref.get() != null) throw ref.get(); } @@ -99,7 +103,7 @@ public class ShutdownIT extends ConfigurableMacBase { @Test public void stopDuringStart() throws Exception { - assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); } @Test @@ -112,13 +116,14 @@ public class ShutdownIT extends ConfigurableMacBase { static void runAdminStopTest(AccumuloClient c, MiniAccumuloClusterImpl cluster) throws InterruptedException, IOException { String confPath = cluster.getConfig().getClientPropsFile().getAbsolutePath(); - int x = cluster.exec(TestIngest.class, "--config-file", confPath, "--createTable").waitFor(); + int x = cluster.exec(TestIngest.class, "--config-file", confPath, "--createTable").getProcess() + .waitFor(); assertEquals(0, x); List tabletServers = c.instanceOperations().getTabletServers(); assertEquals(2, tabletServers.size()); String doomed = tabletServers.get(0); log.info("Stopping " + doomed); - assertEquals(0, cluster.exec(Admin.class, "stop", doomed).waitFor()); + assertEquals(0, cluster.exec(Admin.class, "stop", doomed).getProcess().waitFor()); tabletServers = c.instanceOperations().getTabletServers(); assertEquals(1, tabletServers.size()); assertNotEquals(tabletServers.get(0), doomed); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java index 7fd76b8..b1d154d 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java @@ -132,7 +132,7 @@ public class WALSunnyDayIT extends ConfigurableMacBase { // restart the tserver, but don't run recovery on all tablets control.stop(TABLET_SERVER); // this delays recovery on the normal tables - assertEquals(0, cluster.exec(SetGoalState.class, "SAFE_MODE").waitFor()); + assertEquals(0, cluster.exec(SetGoalState.class, "SAFE_MODE").getProcess().waitFor()); control.start(TABLET_SERVER); // wait for the metadata table to go back online @@ -146,7 +146,7 @@ public class WALSunnyDayIT extends ConfigurableMacBase { markers.keySet().iterator().next().getTableId().canonicalID()); // put some data in the WAL - assertEquals(0, cluster.exec(SetGoalState.class, "NORMAL").waitFor()); + assertEquals(0, cluster.exec(SetGoalState.class, "NORMAL").getProcess().waitFor()); verifySomeData(c, tableName, 1001 * 50 + 1); writeSomeData(c, tableName, 100, 100); diff --git a/test/src/main/java/org/apache/accumulo/test/mapreduce/MapReduceIT.java b/test/src/main/java/org/apache/accumulo/test/mapreduce/MapReduceIT.java index 8f8ff44..ac0b420 100644 --- a/test/src/main/java/org/apache/accumulo/test/mapreduce/MapReduceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/mapreduce/MapReduceIT.java @@ -96,9 +96,11 @@ public class MapReduceIT extends ConfigurableMacBase { c.properties().store(out, "Config file for " + MapReduceIT.class.getName()); } ClientInfo info = ClientInfo.from(c.properties()); - Process hash = cluster.exec(RowHash.class, Collections.singletonList(hadoopTmpDirArg), "-i", - info.getInstanceName(), "-z", info.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "-t", - tablename, "--column", input_cfcq, "--config-file", configFile.getAbsolutePath()); + Process hash = cluster + .exec(RowHash.class, Collections.singletonList(hadoopTmpDirArg), "-i", + info.getInstanceName(), "-z", info.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, + "-t", tablename, "--column", input_cfcq, "--config-file", configFile.getAbsolutePath()) + .getProcess(); assertEquals(0, hash.waitFor()); try (Scanner s = c.createScanner(tablename, Authorizations.EMPTY)) { diff --git a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java index 5ed0573..46097ad 100644 --- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java @@ -1198,7 +1198,7 @@ public class ReplicationIT extends ConfigurableMacBase { // Starting the gc will run CloseWriteAheadLogReferences which will first close Statuses // in the metadata table, and then in the replication table - Process gc = cluster.exec(SimpleGarbageCollector.class); + Process gc = cluster.exec(SimpleGarbageCollector.class).getProcess(); waitForGCLock(client);