From commits-return-29543-archive-asf-public=cust-asf.ponee.io@geode.apache.org Mon Nov 26 23:46:54 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id EEE1A180647 for ; Mon, 26 Nov 2018 23:46:53 +0100 (CET) Received: (qmail 39000 invoked by uid 500); 26 Nov 2018 22:46:53 -0000 Mailing-List: contact commits-help@geode.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.apache.org Delivered-To: mailing list commits@geode.apache.org Received: (qmail 38991 invoked by uid 99); 26 Nov 2018 22:46:53 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 26 Nov 2018 22:46:53 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 6F5C485205; Mon, 26 Nov 2018 22:46:52 +0000 (UTC) Date: Mon, 26 Nov 2018 22:46:52 +0000 To: "commits@geode.apache.org" Subject: [geode-benchmarks] branch develop updated: Use a Java ssh server rather than docker for tests MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <154327241240.13326.14619671863652238025@gitbox.apache.org> From: upthewaterspout@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: geode-benchmarks X-Git-Refname: refs/heads/develop X-Git-Reftype: branch X-Git-Oldrev: 27ce7198f9fd200e5c6fc12fb6e1af90462d7cb1 X-Git-Newrev: 18e309dea04f4e611ef3956165935f694216ad4b X-Git-Rev: 18e309dea04f4e611ef3956165935f694216ad4b X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. upthewaterspout pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git The following commit(s) were added to refs/heads/develop by this push: new 18e309d Use a Java ssh server rather than docker for tests 18e309d is described below commit 18e309dea04f4e611ef3956165935f694216ad4b Author: Dan Smith AuthorDate: Mon Nov 26 14:46:48 2018 -0800 Use a Java ssh server rather than docker for tests To test the ssh infrastructure, use apache sshd in the test to run a ssh server. This makes the tests less dependent on specific infrastructure and build magic. Users can now run the tests on their machines without having to set up passwordless ssh (Although actual benchmark runs will still require it). Cleaning up how SshInfrastructure sends multiple commands. --- .travis.yml | 17 +----- harness/build.gradle | 1 + .../infrastructure/ssh/SshInfrastructure.java | 23 ++++--- .../infrastructure/ssh/SshInfrastructureTest.java | 16 +++-- .../perftest/infrastructure/ssh/SshServerRule.java | 71 ++++++++++++++++++++++ images/geode-performance/Dockerfile | 33 ---------- 6 files changed, 99 insertions(+), 62 deletions(-) diff --git a/.travis.yml b/.travis.yml index 620a982..6fe694f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,5 @@ -required: sudo - language: java -services: - - docker - jdk: - openjdk8 @@ -18,13 +13,5 @@ cache: - $HOME/.gradle/wrapper/ script: - - ./gradlew buildTestingImage - - docker run -p 2222:22 -d --name geode-test geode-performance-testing - - ssh-keygen -N "" -f ./id_rsa - - docker cp id_rsa geode-test:/root/.ssh/id_rsa - - docker exec -it geode-test chown root:root /root/.ssh/id_rsa - - docker cp id_rsa.pub geode-test:/root/.ssh/id_rsa.pub - - docker exec -it geode-test chown root:root /root/.ssh/id_rsa.pub - - docker cp id_rsa.pub geode-test:/root/.ssh/authorized_keys - - docker exec -it geode-test chown root:root /root/.ssh/authorized_keys - - ssh -o "StrictHostKeyChecking=no" -i id_rsa -p 2222 root@localhost "pushd /geode-performance; ./gradlew geode-benchmarks:test" + - ssh-keygen -N "" -f ~/.ssh/id_rsa + - ./gradlew check diff --git a/harness/build.gradle b/harness/build.gradle index 1658ecf..7b32cd7 100644 --- a/harness/build.gradle +++ b/harness/build.gradle @@ -36,4 +36,5 @@ dependencies { testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19' testCompile group: 'org.awaitility', name: 'awaitility', version: '3.0.0' testCompile group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25' + testCompile group: 'org.apache.sshd', name: 'sshd-core', version: '2.1.0' } diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java index 4ff90e4..bff36b1 100644 --- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java +++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java @@ -49,20 +49,25 @@ public class SshInfrastructure implements Infrastructure { private final Set hosts; private final String user; + private final int port; public static final Config CONFIG = new DefaultConfig(); public SshInfrastructure(Collection hosts, String user) { + this(hosts, user, 22); + } + + public SshInfrastructure(Collection hosts, String user, int port) { this.hosts = hosts.stream() .map(SshNode::new) .collect(Collectors.toCollection(LinkedHashSet::new)); this.user = user; + this.port = port; } SSHClient getSSHClient(InetAddress address) throws IOException { SSHClient client = new SSHClient(CONFIG); client.addHostKeyVerifier(new PromiscuousVerifier()); - client.loadKnownHosts(); - client.connect(address); + client.connect(address, port); client.authPublickey(user); return client; } @@ -109,19 +114,22 @@ public class SshInfrastructure implements Infrastructure { for(InetAddress address: uniqueNodes) { futures.add(CompletableFuture.runAsync(() -> { try (SSHClient client = getSSHClient(address)) { - try (Session session = client.startSession()) { client.useCompression(); if(removeExisting) { - session.exec(String.format("/bin/sh -c \"rm -rf '%s'; mkdir -p '%s'\"", destDir, destDir)).join(); - }else { + try (Session session = client.startSession()) { + session.exec(String.format("rm -rf '%s'", destDir)).join(); + } + } + + try (Session session = client.startSession()) { session.exec(String.format("mkdir -p '%s'", destDir)).join(); } + for (File file : files) { logger.info("Copying " + file + " to " + address); client.newSCPFileTransfer().upload(new FileSystemFile(file), destDir); } - } } catch(IOException e) { throw new UncheckedIOException(e); } @@ -133,14 +141,11 @@ public class SshInfrastructure implements Infrastructure { @Override public void copyFromNode(Node node, String directory, File destDir) throws IOException { try (SSHClient client = getSSHClient(node.getAddress())) { - - try (Session session = client.startSession()) { client.useCompression(); destDir.mkdirs(); client.newSCPFileTransfer().download(directory, destDir.getPath()); return; - } } } diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java index 2936e80..c1041cf 100644 --- a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java +++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java @@ -34,14 +34,19 @@ import org.junit.rules.TemporaryFolder; import org.apache.geode.perftest.infrastructure.Infrastructure; public class SshInfrastructureTest { + + private static final Set HOSTS = Collections.singleton("localhost"); private static final String USER = System.getProperty("user.name"); @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public SshServerRule server = new SshServerRule(); + @Test public void canFindNodes() throws IOException { - SshInfrastructure infra = new SshInfrastructure(HOSTS, USER); + SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort()); assertEquals(1, infra.getNodes().size()); } @@ -49,10 +54,11 @@ public class SshInfrastructureTest { @Test public void canExecuteACommandOnNode() throws IOException { - SshInfrastructure infra = new SshInfrastructure(HOSTS, USER); + SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort()); Infrastructure.Node node1 = infra.getNodes().iterator().next(); File folder = temporaryFolder.newFolder(); + folder.mkdirs(); File expectedFile = new File(folder, "somefile.txt").getAbsoluteFile(); int result = infra.onNode(node1, new String[] {"touch", expectedFile.getPath()} ); @@ -63,7 +69,7 @@ public class SshInfrastructureTest { @Test public void copyToNodesPutsFileOnNode() throws IOException, InterruptedException { - SshInfrastructure infra = new SshInfrastructure(HOSTS, USER); + SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort()); File someFile = temporaryFolder.newFile(); File targetFolder = new File(temporaryFolder.newFolder(), "dest"); @@ -78,7 +84,7 @@ public class SshInfrastructureTest { @Test public void copyToNodesCleansDirectory() throws IOException, InterruptedException { - SshInfrastructure infra = new SshInfrastructure(HOSTS, USER); + SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort()); File someFile = temporaryFolder.newFile(); File targetFolder = new File(temporaryFolder.newFolder(), "dest"); @@ -99,7 +105,7 @@ public class SshInfrastructureTest { @Test public void canCopyFilesFromANode() throws IOException, ExecutionException, InterruptedException { - SshInfrastructure infra = new SshInfrastructure(HOSTS, USER); + SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort()); Infrastructure.Node node1 = infra.getNodes().iterator().next(); infra.onNode(node1, new String[] {"mkdir", "-p", "/tmp/foo"}); diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java new file mode 100644 index 0000000..dfc3ca7 --- /dev/null +++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.geode.perftest.infrastructure.ssh; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Paths; + +import org.apache.sshd.server.SshServer; +import org.apache.sshd.server.command.Command; +import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; +import org.apache.sshd.server.shell.ProcessShellCommandFactory; +import org.junit.rules.TemporaryFolder; + +/** + * Rule to run an in process ssh server during a test + * + * This ssh server listens on localhost. It does actually run commands and create + * files on the real filesystem. It accepts connections from any user. + */ +public class SshServerRule extends TemporaryFolder { + + private SshServer sshd; + + @Override + protected void before() throws Throwable { + super.before(); + sshd = SshServer.setUpDefaultServer(); + sshd.setPort(0); + sshd.setHost("localhost"); + sshd.setPublickeyAuthenticator((username, key, session) -> true); + sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(Paths.get(newFolder().getPath(), "hostkey.ser"))); + sshd.setCommandFactory(new UnescapingCommandFactory()); + sshd.start(); + } + + public int getPort() { + return sshd.getPort(); + } + + @Override + protected void after() { + try { + sshd.stop(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private class UnescapingCommandFactory extends ProcessShellCommandFactory { + @Override + public Command createCommand(String command) { + return super.createCommand(command.replace("'", "")); + } + } +} diff --git a/images/geode-performance/Dockerfile b/images/geode-performance/Dockerfile deleted file mode 100644 index 484c8fd..0000000 --- a/images/geode-performance/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM ubuntu -# This should be built from the top level of the repository. -# docker build -t geode-performance-testing-image -f images/geode-performance/Dockerfile -RUN apt update -y && \ - apt install -y openjdk-8-jdk-headless openssh-server -COPY . /geode-performance -RUN mkdir /var/run/sshd && \ - sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ - sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \ - ssh-keygen -N "" -f /root/.ssh/id_rsa && \ - echo "Host localhost" > /root/.ssh/config && \ - echo " StrictHostKeyChecking no" >> /root/.ssh/config && \ - chmod 600 /root/.ssh/config && \ - cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \ - chmod 600 /root/.ssh/authorized_keys && \ - rm -f /geode-performance/Dockerfile -CMD ["/usr/sbin/sshd", "-D", "-E", "/var/log/sshd.log"] -EXPOSE 22