geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From upthewatersp...@apache.org
Subject [geode-benchmarks] branch develop updated: Use a Java ssh server rather than docker for tests
Date Mon, 26 Nov 2018 22:46:52 GMT
This is an automated email from the ASF dual-hosted git repository.

upthewaterspout pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git


The following commit(s) were added to refs/heads/develop by this push:
     new 18e309d  Use a Java ssh server rather than docker for tests
18e309d is described below

commit 18e309dea04f4e611ef3956165935f694216ad4b
Author: Dan Smith <dsmith@pivotal.io>
AuthorDate: Mon Nov 26 14:46:48 2018 -0800

    Use a Java ssh server rather than docker for tests
    
    To test the ssh infrastructure, use apache sshd in the test to run a ssh
    server.
    
    This makes the tests less dependent on specific infrastructure and build
    magic. Users can now run the tests on their machines without having to
    set up passwordless ssh (Although actual benchmark runs will still
    require it).
    
    Cleaning up how SshInfrastructure sends multiple commands.
---
 .travis.yml                                        | 17 +-----
 harness/build.gradle                               |  1 +
 .../infrastructure/ssh/SshInfrastructure.java      | 23 ++++---
 .../infrastructure/ssh/SshInfrastructureTest.java  | 16 +++--
 .../perftest/infrastructure/ssh/SshServerRule.java | 71 ++++++++++++++++++++++
 images/geode-performance/Dockerfile                | 33 ----------
 6 files changed, 99 insertions(+), 62 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 620a982..6fe694f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,10 +1,5 @@
-required: sudo
-
 language: java
 
-services:
-  - docker
-
 jdk:
   - openjdk8
 
@@ -18,13 +13,5 @@ cache:
     - $HOME/.gradle/wrapper/
 
 script:
-  - ./gradlew buildTestingImage
-  - docker run -p 2222:22 -d --name geode-test geode-performance-testing
-  - ssh-keygen -N "" -f ./id_rsa
-  - docker cp id_rsa geode-test:/root/.ssh/id_rsa
-  - docker exec -it geode-test chown root:root /root/.ssh/id_rsa
-  - docker cp id_rsa.pub geode-test:/root/.ssh/id_rsa.pub
-  - docker exec -it geode-test chown root:root /root/.ssh/id_rsa.pub
-  - docker cp id_rsa.pub geode-test:/root/.ssh/authorized_keys
-  - docker exec -it geode-test chown root:root /root/.ssh/authorized_keys
-  - ssh  -o "StrictHostKeyChecking=no" -i id_rsa -p 2222 root@localhost "pushd /geode-performance;
./gradlew geode-benchmarks:test"
+ - ssh-keygen -N "" -f ~/.ssh/id_rsa
+ - ./gradlew check
diff --git a/harness/build.gradle b/harness/build.gradle
index 1658ecf..7b32cd7 100644
--- a/harness/build.gradle
+++ b/harness/build.gradle
@@ -36,4 +36,5 @@ dependencies {
     testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19'
     testCompile group: 'org.awaitility', name: 'awaitility', version: '3.0.0'
     testCompile group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25'
+    testCompile group: 'org.apache.sshd', name: 'sshd-core', version: '2.1.0'
 }
diff --git a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
index 4ff90e4..bff36b1 100644
--- a/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
+++ b/harness/src/main/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructure.java
@@ -49,20 +49,25 @@ public class SshInfrastructure implements Infrastructure {
 
   private final Set<SshNode> hosts;
   private final String user;
+  private final int port;
   public static final Config CONFIG = new DefaultConfig();
   
   public SshInfrastructure(Collection<String> hosts, String user) {
+    this(hosts, user, 22);
+  }
+
+  public SshInfrastructure(Collection<String> hosts, String user, int port) {
     this.hosts = hosts.stream()
         .map(SshNode::new)
         .collect(Collectors.toCollection(LinkedHashSet::new));
     this.user = user;
+    this.port = port;
   }
 
   SSHClient getSSHClient(InetAddress address) throws IOException {
     SSHClient client = new SSHClient(CONFIG);
     client.addHostKeyVerifier(new PromiscuousVerifier());
-    client.loadKnownHosts();
-    client.connect(address);
+    client.connect(address, port);
     client.authPublickey(user);
     return client;
   }
@@ -109,19 +114,22 @@ public class SshInfrastructure implements Infrastructure {
     for(InetAddress address: uniqueNodes) {
       futures.add(CompletableFuture.runAsync(() -> {
         try (SSHClient client = getSSHClient(address)) {
-          try (Session session = client.startSession()) {
             client.useCompression();
 
             if(removeExisting) {
-              session.exec(String.format("/bin/sh -c \"rm -rf '%s'; mkdir -p '%s'\"", destDir,
destDir)).join();
-            }else {
+              try (Session session = client.startSession()) {
+                session.exec(String.format("rm -rf '%s'", destDir)).join();
+              }
+            }
+
+            try (Session session = client.startSession()) {
               session.exec(String.format("mkdir -p '%s'", destDir)).join();
             }
+
             for (File file : files) {
               logger.info("Copying " + file + " to " + address);
               client.newSCPFileTransfer().upload(new FileSystemFile(file), destDir);
             }
-          }
         } catch(IOException e) {
           throw new UncheckedIOException(e);
         }
@@ -133,14 +141,11 @@ public class SshInfrastructure implements Infrastructure {
   @Override
   public void copyFromNode(Node node, String directory, File destDir) throws IOException
{
     try (SSHClient client = getSSHClient(node.getAddress())) {
-
-      try (Session session = client.startSession()) {
         client.useCompression();
 
         destDir.mkdirs();
         client.newSCPFileTransfer().download(directory, destDir.getPath());
         return;
-      }
     }
 
   }
diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
index 2936e80..c1041cf 100644
--- a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshInfrastructureTest.java
@@ -34,14 +34,19 @@ import org.junit.rules.TemporaryFolder;
 import org.apache.geode.perftest.infrastructure.Infrastructure;
 
 public class SshInfrastructureTest {
+
+
   private static final Set<String> HOSTS = Collections.singleton("localhost");
   private static final String USER = System.getProperty("user.name");
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
+  @Rule
+  public SshServerRule server = new SshServerRule();
+
   @Test
   public void canFindNodes() throws IOException {
-    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
 
     assertEquals(1, infra.getNodes().size());
   }
@@ -49,10 +54,11 @@ public class SshInfrastructureTest {
   @Test
   public void canExecuteACommandOnNode()
       throws IOException {
-    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
     Infrastructure.Node node1 = infra.getNodes().iterator().next();
 
     File folder = temporaryFolder.newFolder();
+    folder.mkdirs();
     File expectedFile = new File(folder, "somefile.txt").getAbsoluteFile();
     int result = infra.onNode(node1, new String[] {"touch", expectedFile.getPath()}
     );
@@ -63,7 +69,7 @@ public class SshInfrastructureTest {
 
   @Test
   public void copyToNodesPutsFileOnNode() throws IOException, InterruptedException {
-    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
 
     File someFile = temporaryFolder.newFile();
     File targetFolder = new File(temporaryFolder.newFolder(), "dest");
@@ -78,7 +84,7 @@ public class SshInfrastructureTest {
 
   @Test
   public void copyToNodesCleansDirectory() throws IOException, InterruptedException {
-    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
 
     File someFile = temporaryFolder.newFile();
     File targetFolder = new File(temporaryFolder.newFolder(), "dest");
@@ -99,7 +105,7 @@ public class SshInfrastructureTest {
   @Test
   public void canCopyFilesFromANode()
       throws IOException, ExecutionException, InterruptedException {
-    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER);
+    SshInfrastructure infra = new SshInfrastructure(HOSTS, USER, server.getPort());
     Infrastructure.Node node1 = infra.getNodes().iterator().next();
 
     infra.onNode(node1, new String[] {"mkdir", "-p", "/tmp/foo"});
diff --git a/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java
b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java
new file mode 100644
index 0000000..dfc3ca7
--- /dev/null
+++ b/harness/src/test/java/org/apache/geode/perftest/infrastructure/ssh/SshServerRule.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.geode.perftest.infrastructure.ssh;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Paths;
+
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.command.Command;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.server.shell.ProcessShellCommandFactory;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Rule to run an in process ssh server during a test
+ *
+ * This ssh server listens on localhost. It does actually run commands and create
+ * files on the real filesystem. It accepts connections from any user.
+ */
+public class SshServerRule extends TemporaryFolder {
+
+  private SshServer sshd;
+
+  @Override
+  protected void before() throws Throwable {
+    super.before();
+    sshd = SshServer.setUpDefaultServer();
+    sshd.setPort(0);
+    sshd.setHost("localhost");
+    sshd.setPublickeyAuthenticator((username, key, session) -> true);
+    sshd.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(Paths.get(newFolder().getPath(),
"hostkey.ser")));
+    sshd.setCommandFactory(new UnescapingCommandFactory());
+    sshd.start();
+  }
+
+  public int getPort() {
+    return sshd.getPort();
+  }
+
+  @Override
+  protected void after() {
+    try {
+      sshd.stop();
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  private class UnescapingCommandFactory extends ProcessShellCommandFactory {
+    @Override
+    public Command createCommand(String command) {
+      return super.createCommand(command.replace("'", ""));
+    }
+  }
+}
diff --git a/images/geode-performance/Dockerfile b/images/geode-performance/Dockerfile
deleted file mode 100644
index 484c8fd..0000000
--- a/images/geode-performance/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM ubuntu
-# This should be built from the top level of the repository.
-# docker build -t geode-performance-testing-image -f images/geode-performance/Dockerfile
-RUN apt update -y && \
-  apt install -y openjdk-8-jdk-headless openssh-server
-COPY . /geode-performance
-RUN mkdir /var/run/sshd && \
-  sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
&& \
-  sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
&& \
-  ssh-keygen -N "" -f /root/.ssh/id_rsa && \
-  echo "Host localhost" > /root/.ssh/config && \
-  echo "    StrictHostKeyChecking no" >> /root/.ssh/config && \
-  chmod 600 /root/.ssh/config && \
-  cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
-  chmod 600 /root/.ssh/authorized_keys && \
-  rm -f /geode-performance/Dockerfile
-CMD ["/usr/sbin/sshd", "-D", "-E", "/var/log/sshd.log"]
-EXPOSE 22


Mime
View raw message