airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dimuthu...@apache.org
Subject [airavata] branch develop updated: Load generating client initial version
Date Thu, 02 Aug 2018 13:40:13 GMT
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git


The following commit(s) were added to refs/heads/develop by this push:
     new 6118d04  Load generating client initial version
6118d04 is described below

commit 6118d045b43f6d630770e2e41b096302af53d05e
Author: dimuthu <dimuthu.upeksha2@gmail.com>
AuthorDate: Thu Aug 2 09:40:01 2018 -0400

    Load generating client initial version
---
 .../apache/airavata/agents/api/AgentAdaptor.java   |   2 +
 .../airavata/helix/agent/ssh/SshAgentAdaptor.java  |   5 +
 .../airavata/helix/adaptor/SSHJAgentAdaptor.java   |  14 ++
 pom.xml                                            |   1 +
 tools/gsissh-cli-tools/pom.xml                     |   2 +-
 tools/load-client/pom.xml                          |  76 +++++++++
 .../src/main/assembly/load-client-bin-assembly.xml |  82 +++++++++
 .../apache/airavata/tools/load/Configuration.java  | 186 +++++++++++++++++++++
 .../apache/airavata/tools/load/Configurations.java |  17 ++
 .../org/apache/airavata/tools/load/LoadClient.java | 140 ++++++++++++++++
 .../airavata/tools/load/SecurityManager.java       |  72 ++++++++
 .../tools/load/StorageResourceManager.java         |  96 +++++++++++
 .../org/apache/airavata/tools/load/UnitLoad.java   | 150 +++++++++++++++++
 .../src/main/resources/bin/load-client.sh          |  76 +++++++++
 tools/load-client/src/main/resources/bin/setenv.sh |  46 +++++
 .../src/main/resources/client_truststore.jks       | Bin 0 -> 5312 bytes
 .../src/main/resources/conf/load-config.yml        |  22 +++
 tools/phoebus-integration/pom.xml                  |   2 +-
 tools/pom.xml                                      |   5 +-
 19 files changed, 989 insertions(+), 5 deletions(-)

diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
index bbeaf92..5355d5c 100644
--- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
+++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
@@ -32,6 +32,8 @@ public interface AgentAdaptor {
 
     public void init(String computeResource, String gatewayId, String userId, String token)
throws AgentException;
 
+    public void destroy();
+
     public CommandOutput executeCommand(String command, String workingDirectory) throws AgentException;
 
     public void createDirectory(String path) throws AgentException;
diff --git a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
index ffa9109c..e4cffe9 100644
--- a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
+++ b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
@@ -106,6 +106,11 @@ public class SshAgentAdaptor implements AgentAdaptor {
         }
     }
 
+    @Override
+    public void destroy() {
+
+    }
+
     public CommandOutput executeCommand(String command, String workingDirectory) throws AgentException
{
         StandardOutReader commandOutput = new StandardOutReader();
         ChannelExec channelExec = null;
diff --git a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
index 4349844..791724a 100644
--- a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
+++ b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
@@ -128,6 +128,20 @@ public class SSHJAgentAdaptor implements AgentAdaptor {
     }
 
     @Override
+    public void destroy() {
+        try {
+            if (sshjClient != null) {
+                sshjClient.disconnect();
+                sshjClient.close();
+            }
+        } catch (IOException e) {
+            logger.warn("Failed to stop sshj client for host " + sshjClient.getHost() + "
and user " +
+                    sshjClient.getUsername() + " due to : " + e.getMessage());
+            // ignore
+        }
+    }
+
+    @Override
     public CommandOutput executeCommand(String command, String workingDirectory) throws AgentException
{
         try (Session session = sshjClient.startSessionWrapper()) {
 
diff --git a/pom.xml b/pom.xml
index 29caf3f..d554eae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -770,6 +770,7 @@
                 <module>modules/compute-account-provisioning</module>
                 <module>modules/airavata-helix</module>
                 <module>modules/job-monitor</module>
+                <module>tools</module>
             </modules>
         </profile>
         <profile>
diff --git a/tools/gsissh-cli-tools/pom.xml b/tools/gsissh-cli-tools/pom.xml
index 0c5b020..b99f1ad 100644
--- a/tools/gsissh-cli-tools/pom.xml
+++ b/tools/gsissh-cli-tools/pom.xml
@@ -26,7 +26,7 @@
 	<parent>
 		<groupId>org.apache.airavata</groupId>
 		<artifactId>airavata</artifactId>
-		<version>0.14-SNAPSHOT</version>
+		<version>0.17-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 
diff --git a/tools/load-client/pom.xml b/tools/load-client/pom.xml
new file mode 100644
index 0000000..56ac6a4
--- /dev/null
+++ b/tools/load-client/pom.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>org.apache.airavata</groupId>
+        <artifactId>airavata-tools-parent</artifactId>
+        <version>0.17-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>load-client</artifactId>
+    <name>Airavata Load Testing Client</name>
+    <description>Puts a load to Airavata through API</description>
+    <url>http://airavata.apache.org/</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>airavata-api-stubs</artifactId>
+            <version>0.17-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jboss.resteasy</groupId>
+            <artifactId>resteasy-client</artifactId>
+            <version>3.0.14.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>org.keycloak</groupId>
+            <artifactId>keycloak-admin-client</artifactId>
+            <version>2.5.5.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>sshj-agent</artifactId>
+            <version>0.17-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.15</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.2</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.5.5</version>
+                <executions>
+                    <execution>
+                        <id>load-client-distribution-package</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <tarLongFileMode>posix</tarLongFileMode>
+                            <finalName>load-client-${project.version}</finalName>
+                            <descriptors>
+                                <descriptor>src/main/assembly/load-client-bin-assembly.xml</descriptor>
+                            </descriptors>
+                            <attach>false</attach>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file
diff --git a/tools/load-client/src/main/assembly/load-client-bin-assembly.xml b/tools/load-client/src/main/assembly/load-client-bin-assembly.xml
new file mode 100644
index 0000000..60d7f7f
--- /dev/null
+++ b/tools/load-client/src/main/assembly/load-client-bin-assembly.xml
@@ -0,0 +1,82 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<!DOCTYPE assembly [
+        <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+        <!ELEMENT id (#PCDATA)>
+        <!ELEMENT includeBaseDirectory (#PCDATA)>
+        <!ELEMENT baseDirectory (#PCDATA)>
+        <!ELEMENT formats (format)*>
+        <!ELEMENT format (#PCDATA)>
+        <!ELEMENT fileSets (fileSet)*>
+        <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+        <!ELEMENT directory (#PCDATA)>
+        <!ELEMENT outputDirectory (#PCDATA)>
+        <!ELEMENT includes (include)*>
+        <!ELEMENT include (#PCDATA)>
+        <!ELEMENT dependencySets (dependencySet)*>
+        <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+        ]>
+<assembly>
+    <id>bin</id>
+    <includeBaseDirectory>true</includeBaseDirectory>
+    <baseDirectory>load-client</baseDirectory>
+    <formats>
+        <format>tar.gz</format>
+        <format>zip</format>
+    </formats>
+
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources/bin</directory>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>777</fileMode>
+            <includes>
+                <include>load-client.sh</include>
+                <include>setenv.sh</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>777</fileMode>
+            <includes>
+                <include>client_truststore.jks</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>src/main/resources/conf</directory>
+            <outputDirectory>conf</outputDirectory>
+            <includes>
+                <include>load-config.yml</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>true</useProjectArtifact>
+            <outputDirectory>lib</outputDirectory>
+            <includes>
+                <include>*:*:jar</include>
+            </includes>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configuration.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configuration.java
new file mode 100644
index 0000000..c8484ea
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configuration.java
@@ -0,0 +1,186 @@
+package org.apache.airavata.tools.load;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Configuration {
+
+    private String userId;
+
+    private String gatewayId;
+    private String projectId;
+    private String applicationInterfaceId;
+    private String computeResourceId;
+    private String storageResourceId;
+
+    private String experimentBaseName;
+
+    private String queue;
+    private int wallTime;
+    private int cpuCount;
+    private int nodeCount;
+    private int physicalMemory;
+
+    private int concurrentUsers;
+    private int iterationsPerUser;
+    private int randomMSDelayWithinSubmissions;
+
+    private List<Input> inputs = new ArrayList<>();
+
+    public String getUserId() {
+        return userId;
+    }
+
+    public void setUserId(String userId) {
+        this.userId = userId;
+    }
+
+    public String getGatewayId() {
+        return gatewayId;
+    }
+
+    public void setGatewayId(String gatewayId) {
+        this.gatewayId = gatewayId;
+    }
+
+    public String getProjectId() {
+        return projectId;
+    }
+
+    public void setProjectId(String projectId) {
+        this.projectId = projectId;
+    }
+
+    public String getApplicationInterfaceId() {
+        return applicationInterfaceId;
+    }
+
+    public void setApplicationInterfaceId(String applicationInterfaceId) {
+        this.applicationInterfaceId = applicationInterfaceId;
+    }
+
+    public String getComputeResourceId() {
+        return computeResourceId;
+    }
+
+    public void setComputeResourceId(String computeResourceId) {
+        this.computeResourceId = computeResourceId;
+    }
+
+    public String getStorageResourceId() {
+        return storageResourceId;
+    }
+
+    public void setStorageResourceId(String storageResourceId) {
+        this.storageResourceId = storageResourceId;
+    }
+
+    public String getExperimentBaseName() {
+        return experimentBaseName;
+    }
+
+    public void setExperimentBaseName(String experimentBaseName) {
+        this.experimentBaseName = experimentBaseName;
+    }
+
+    public String getQueue() {
+        return queue;
+    }
+
+    public void setQueue(String queue) {
+        this.queue = queue;
+    }
+
+    public int getWallTime() {
+        return wallTime;
+    }
+
+    public void setWallTime(int wallTime) {
+        this.wallTime = wallTime;
+    }
+
+    public int getCpuCount() {
+        return cpuCount;
+    }
+
+    public void setCpuCount(int cpuCount) {
+        this.cpuCount = cpuCount;
+    }
+
+    public int getNodeCount() {
+        return nodeCount;
+    }
+
+    public void setNodeCount(int nodeCount) {
+        this.nodeCount = nodeCount;
+    }
+
+    public int getPhysicalMemory() {
+        return physicalMemory;
+    }
+
+    public void setPhysicalMemory(int physicalMemory) {
+        this.physicalMemory = physicalMemory;
+    }
+
+    public int getConcurrentUsers() {
+        return concurrentUsers;
+    }
+
+    public void setConcurrentUsers(int concurrentUsers) {
+        this.concurrentUsers = concurrentUsers;
+    }
+
+    public int getIterationsPerUser() {
+        return iterationsPerUser;
+    }
+
+    public void setIterationsPerUser(int iterationsPerUser) {
+        this.iterationsPerUser = iterationsPerUser;
+    }
+
+    public int getRandomMSDelayWithinSubmissions() {
+        return randomMSDelayWithinSubmissions;
+    }
+
+    public void setRandomMSDelayWithinSubmissions(int randomMSDelayWithinSubmissions) {
+        this.randomMSDelayWithinSubmissions = randomMSDelayWithinSubmissions;
+    }
+
+    public List<Input> getInputs() {
+        return inputs;
+    }
+
+    public void setInputs(List<Input> inputs) {
+        this.inputs = inputs;
+    }
+
+    public static class Input {
+        private String name;
+        private String value;
+
+        public Input() {
+        }
+
+        public Input(String name, String value) {
+            this.name = name;
+            this.value = value;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public String getValue() {
+            return value;
+        }
+
+        public void setValue(String value) {
+            this.value = value;
+        }
+    }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configurations.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configurations.java
new file mode 100644
index 0000000..b267f1f
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/Configurations.java
@@ -0,0 +1,17 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.tools.load.Configuration;
+
+import java.util.List;
+
+public class Configurations {
+    private List<Configuration> configurations;
+
+    public List<Configuration> getConfigurations() {
+        return configurations;
+    }
+
+    public void setConfigurations(List<Configuration> configurations) {
+        this.configurations = configurations;
+    }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
new file mode 100644
index 0000000..283499a
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
@@ -0,0 +1,140 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference;
+import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.commons.cli.*;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletionService;
+
+public class LoadClient {
+
+    private String apiHost;
+    private int apiPort = 9930;
+
+    private String privateKeyFile = System.getProperty("user.home") + "/.ssh/id_rsa";
+    private String publicKeyFile = System.getProperty("user.home") + "/.ssh/id_rsa.pub";
+    private String passPhrase = null;
+    private String configFile;
+
+    private SecurityManager securityManager = new SecurityManager();
+    private Map<String, StorageResourceManager> storageResourceManagerStore = new HashMap<>();
+    private Configurations configurations;
+
+    public void init() throws Exception {
+        securityManager.loadCertificate(apiHost, apiPort);
+
+        if (configFile == null) {
+            try (InputStream in = LoadClient.class.getResourceAsStream("/conf/load-config.yml"))
{
+                Yaml yaml = new Yaml();
+                configurations = yaml.loadAs(in, Configurations.class);
+                createStorageResourceManagers(configurations);
+            }
+        } else {
+            try (InputStream in = new FileInputStream(configFile)) {
+                Yaml yaml = new Yaml();
+                configurations = yaml.loadAs(in, Configurations.class);
+                createStorageResourceManagers(configurations);
+            }
+        }
+    }
+
+    public void start() throws Exception {
+        for (Configuration configuration : configurations.getConfigurations()) {
+            UnitLoad unitLoad = new UnitLoad(apiHost, apiPort, securityManager.getTrustStorePath(),
securityManager.getTrustStorePassword(),
+                    storageResourceManagerStore.get(configuration.getStorageResourceId()));
+            CompletionService<Boolean> completion = unitLoad.execute(configuration);
+
+            for (int i = 0; i < configuration.getConcurrentUsers(); i++) {
+                completion.take();
+            }
+        }
+        destroyStorageResourceManagers();
+        System.out.println("Finished load");
+        System.exit(0);
+    }
+
+    private void createStorageResourceManagers(Configurations configurations) throws Exception
{
+
+        Airavata.Client airavataClient = AiravataClientFactory.createAiravataSecureClient(apiHost,
apiPort,
+                securityManager.getTrustStorePath(), securityManager.getTrustStorePassword(),
100000);
+
+        for (Configuration configuration : configurations.getConfigurations()) {
+            String storageResourceId = configuration.getStorageResourceId();
+
+            if (!storageResourceManagerStore.containsKey(storageResourceId)) {
+                StorageResourceDescription storageResource = airavataClient.getStorageResource(new
AuthzToken(""), storageResourceId);
+                StoragePreference gatewayStoragePreference = airavataClient.getGatewayStoragePreference(new
AuthzToken(""), configuration.getGatewayId(), storageResourceId);
+                StorageResourceManager storageResourceManager = new StorageResourceManager(gatewayStoragePreference,
storageResource, privateKeyFile, publicKeyFile, passPhrase);
+                storageResourceManager.init();
+                storageResourceManagerStore.put(storageResourceId, storageResourceManager);
+            }
+        }
+    }
+
+    private void destroyStorageResourceManagers() {
+        storageResourceManagerStore.values().forEach(StorageResourceManager::destroy);
+    }
+
+    public static void main(String args[]) throws Exception {
+
+        Options options = new Options();
+        options.addOption("config", true, "Load configuration file in yaml format");
+        options.addOption("apiHost", true, "API Server host name");
+        options.addOption("apiPort", true, "API Server port");
+        options.addOption("privateKeyPath", true, "SSH private key path to communicate with
storage resources (Defaults to user private key in ~/.ssh/id_rsa)");
+        options.addOption("publicKeyPath", true, "SSH public key path to communicate with
storage resources (Defaults to user public key in ~/.ssh/id_rsa.pub)");
+        options.addOption("passPhrase", true, "SSH private key pass phrase (if any)");
+
+        CommandLineParser parser = new GnuParser();
+        CommandLine cmd = parser.parse( options, args);
+
+        LoadClient loadClient = new LoadClient();
+
+        if (cmd.hasOption("config")) {
+            loadClient.configFile = cmd.getOptionValue("config");
+        } else {
+            System.out.println("Error : Load config file should be specified");
+            System.exit(0);
+        }
+
+        if (cmd.hasOption("apiHost")) {
+            loadClient.apiHost = cmd.getOptionValue("apiHost");
+        } else {
+            System.out.println("Error : API host should be specified");
+            System.exit(0);
+        }
+
+        if (cmd.hasOption("apiPort")) {
+            loadClient.apiPort = Integer.parseInt(cmd.getOptionValue("apiPort"));
+        } else {
+            System.out.println("Using default API port " + loadClient.apiPort);
+        }
+
+        if (cmd.hasOption("privateKeyPath")) {
+            loadClient.privateKeyFile = cmd.getOptionValue("privateKeyPath");
+        } else {
+            System.out.println("Using default private key file " + loadClient.privateKeyFile);
+        }
+
+        if (cmd.hasOption("publicKeyPath")) {
+            loadClient.publicKeyFile = cmd.getOptionValue("publicKeyPath");
+        } else {
+            System.out.println("Using default public key file " + loadClient.publicKeyFile);
+        }
+
+        if (cmd.hasOption("passPhrase")) {
+            loadClient.passPhrase = cmd.getOptionValue("passPhrase");
+        }
+
+        loadClient.init();
+        loadClient.start();
+    }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/SecurityManager.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/SecurityManager.java
new file mode 100644
index 0000000..410c2af
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/SecurityManager.java
@@ -0,0 +1,72 @@
+package org.apache.airavata.tools.load;
+
+import javax.net.ssl.*;
+import java.io.*;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+
+public class SecurityManager {
+
+    private String trustStoreName = "client_truststore.jks";
+    private String trustStorePassword = "airavata";
+
+    public void loadCertificate(String host, int port) throws CertificateException, NoSuchAlgorithmException,
IOException, KeyStoreException, KeyManagementException, URISyntaxException {
+
+        TrustManager[] trustAllCerts = new TrustManager[]{
+                new X509TrustManager() {
+                    public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+                        return null;
+                    }
+                    public void checkClientTrusted(
+                            java.security.cert.X509Certificate[] certs, String authType)
{
+                    }
+                    public void checkServerTrusted(
+                            java.security.cert.X509Certificate[] certs, String authType)
{
+                    }
+                }
+        };
+
+        SSLContext sslContext = SSLContext.getInstance("SSL");
+        sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
+        SSLSocket socket = (SSLSocket) sslContext.getSocketFactory().createSocket(host, port);
+        socket.startHandshake();
+        SSLSession sslSession = socket.getSession();
+        Certificate[] certificates = sslSession.getPeerCertificates();
+
+        FileInputStream is = new FileInputStream(getTrustStorePath());
+
+        KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType());
+        keystore.load(is, trustStorePassword.toCharArray());
+        is.close();
+
+        File keystoreFile = new File(getTrustStorePath());
+
+        String certificateAlias = host;
+        keystore.setCertificateEntry(certificateAlias, certificates[0]);
+
+        FileOutputStream out = new FileOutputStream(keystoreFile);
+        keystore.store(out, trustStorePassword.toCharArray());
+        out.close();
+
+        System.out.println("Certificates successfully loaded for " + host + ":" + port);
+    }
+
+    public String getTrustStorePath() throws URISyntaxException {
+        URL trustStoreUrl = SecurityManager.class.getClassLoader().getResource(trustStoreName);
+
+        String trustStorePath;
+        if (trustStoreUrl.toURI().getPath() != null) {
+            trustStorePath = trustStoreUrl.toURI().getPath();
+        } else {
+            trustStorePath = System.getProperty("airavata.home") + "/bin/" + trustStoreName;
+        }
+        return trustStorePath;
+    }
+
+    public String getTrustStorePassword() {
+        return trustStorePassword;
+    }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/StorageResourceManager.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/StorageResourceManager.java
new file mode 100644
index 0000000..7f7527f
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/StorageResourceManager.java
@@ -0,0 +1,96 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.helix.adaptor.SSHJStorageAdaptor;
+import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference;
+import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
+import org.apache.airavata.model.data.replica.*;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.thrift.TException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+
+public class StorageResourceManager {
+
+    private StoragePreference gatewayStoragePreference;
+    private StorageResourceDescription storageResource;
+    private String storageResourceId;
+
+    private String privateKeyFile;
+    private String publicKeyFile;
+    private String passPhrase;
+
+    private SSHJStorageAdaptor storageAdaptor = new SSHJStorageAdaptor();
+
+    public StorageResourceManager(StoragePreference gatewayStoragePreference, StorageResourceDescription
storageResource,
+                                  String privateKeyFile, String publicKeyFile, String passPhrase)
{
+        this.storageResourceId = storageResource.getStorageResourceId();
+        this.storageResource = storageResource;
+        this.gatewayStoragePreference = gatewayStoragePreference;
+        this.privateKeyFile = privateKeyFile;
+        this.publicKeyFile = publicKeyFile;
+        this.passPhrase = passPhrase;
+    }
+
+    public void init() throws IOException, AgentException {
+        storageAdaptor.init(gatewayStoragePreference.getLoginUserName(),
+                storageResource.getHostName(),
+                readFile(publicKeyFile, Charset.defaultCharset()),
+                readFile(privateKeyFile, Charset.defaultCharset()),
+                passPhrase);
+    }
+
+    public void destroy() {
+        storageAdaptor.destroy();
+    }
+
+    public String uploadInputFile(Airavata.Client airavataClient, String filePath, String
user, String project, String experiment, String gatewayId) throws TException, AgentException
{
+
+        String experimentDirectory = getExperimentDirectory(user, project, experiment);
+
+        String uploadFilePath = experimentDirectory.concat(File.separator).concat(new File(filePath).getName());
+        storageAdaptor.uploadFile(filePath, uploadFilePath);
+
+        DataProductModel dataProductModel = new DataProductModel();
+        dataProductModel.setGatewayId(gatewayId);
+        dataProductModel.setOwnerName(user);
+        dataProductModel.setDataProductType(DataProductType.FILE);
+
+        DataReplicaLocationModel replicaLocationModel = new DataReplicaLocationModel();
+        replicaLocationModel.setStorageResourceId(storageResourceId);
+        replicaLocationModel.setReplicaName((new File(filePath).getName()) + " gateway data
store copy");
+        replicaLocationModel.setReplicaLocationCategory(ReplicaLocationCategory.GATEWAY_DATA_STORE);
+        replicaLocationModel.setReplicaPersistentType(ReplicaPersistentType.TRANSIENT);
+        replicaLocationModel.setFilePath("file://" + storageResource.getHostName() + ":"
+ uploadFilePath);
+
+        dataProductModel.setReplicaLocations(Collections.singletonList(replicaLocationModel));
+        System.out.println("Registring " +  uploadFilePath);
+        return airavataClient.registerDataProduct(new AuthzToken(""), dataProductModel);
+    }
+
+    public void createExperimentDirectory(String user, String project, String experiment)
throws AgentException {
+        String experimentDirectory = getExperimentDirectory(user, project, experiment);
+        storageAdaptor.createDirectory(experimentDirectory, true);
+    }
+
+    private String getExperimentDirectory(String user, String project, String experiment)
{
+        return gatewayStoragePreference.getFileSystemRootLocation()
+                .concat(File.separator)
+                .concat(user)
+                .concat(File.separator)
+                .concat(project)
+                .concat(File.separator)
+                .concat(experiment);
+    }
+
+    static String readFile(String path, Charset encoding) throws IOException {
+        byte[] encoded = Files.readAllBytes(Paths.get(path));
+        return new String(encoded, encoding);
+    }
+}
diff --git a/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
new file mode 100644
index 0000000..97e7496
--- /dev/null
+++ b/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
@@ -0,0 +1,150 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.agents.api.AgentException;
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.model.application.io.DataType;
+import org.apache.airavata.model.application.io.InputDataObjectType;
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.experiment.ExperimentType;
+import org.apache.airavata.model.experiment.UserConfigurationDataModel;
+import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.airavata.tools.load.Configuration;
+import org.apache.airavata.tools.load.StorageResourceManager;
+import org.apache.thrift.TException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.*;
+
+public class UnitLoad {
+
+    private String apiHost;
+    private int apiPort;
+    private String trustStorePath;
+    private String trustStorePassword;
+    private StorageResourceManager storageResourceManager;
+
+    public UnitLoad(String apiHost, int apiPort, String trustStorePath, String trustStorePassword,
StorageResourceManager storageResourceManager) {
+        this.apiHost = apiHost;
+        this.apiPort = apiPort;
+        this.trustStorePath = trustStorePath;
+        this.trustStorePassword = trustStorePassword;
+        this.storageResourceManager = storageResourceManager;
+    }
+
+    public CompletionService<Boolean> execute(Configuration config) {
+        String randomUUID = UUID.randomUUID().toString();
+        ExecutorService executorService = Executors.newFixedThreadPool(config.getConcurrentUsers());
+        CompletionService<Boolean> completionService = new ExecutorCompletionService<>(executorService);
+
+        for (int i = 0; i < config.getConcurrentUsers(); i++) {
+            completionService.submit(new Worker(config, randomUUID + "-" + i, config.getIterationsPerUser(),
config.getRandomMSDelayWithinSubmissions()));
+        }
+        return completionService;
+    }
+
+    public class Worker implements Callable<Boolean> {
+
+        private final String id;
+        private final int iterations;
+        private final int delay;
+        private final Configuration config;
+
+        public Worker(Configuration config, String id, int iterations, int delay) {
+            this.id = id;
+            this.iterations = iterations;
+            this.delay = delay;
+            this.config = config;
+        }
+
+        @Override
+        public Boolean call() {
+            for (int i = 0; i < iterations; i++) {
+                try {
+                    submitExperiment(config,id + "-" + i);
+                    Thread.sleep(delay);
+                } catch (TException e) {
+                    e.printStackTrace();
+                } catch (AgentException e) {
+                    e.printStackTrace();
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+            return true;
+        }
+    }
+
+    private void submitExperiment(Configuration config, String suffix) throws TException,
AgentException {
+
+        String experimentName = config.getExperimentBaseName() + suffix;
+
+        ExperimentModel experimentModel = new ExperimentModel();
+        experimentModel.setExperimentName(experimentName);
+        experimentModel.setProjectId(config.getProjectId());
+        experimentModel.setUserName(config.getUserId());
+        experimentModel.setGatewayId(config.getGatewayId());
+        experimentModel.setExecutionId(config.getApplicationInterfaceId());
+
+        ComputationalResourceSchedulingModel computationalResourceSchedulingModel = new ComputationalResourceSchedulingModel();
+        computationalResourceSchedulingModel.setQueueName(config.getQueue());
+        computationalResourceSchedulingModel.setNodeCount(config.getNodeCount());
+        computationalResourceSchedulingModel.setTotalCPUCount(config.getCpuCount());
+        computationalResourceSchedulingModel.setWallTimeLimit(config.getWallTime());
+        computationalResourceSchedulingModel.setTotalPhysicalMemory(config.getPhysicalMemory());
+        computationalResourceSchedulingModel.setResourceHostId(config.getComputeResourceId());
+
+        UserConfigurationDataModel userConfigurationDataModel = new UserConfigurationDataModel();
+        userConfigurationDataModel.setComputationalResourceScheduling(computationalResourceSchedulingModel);
+        userConfigurationDataModel.setAiravataAutoSchedule(false);
+        userConfigurationDataModel.setOverrideManualScheduledParams(false);
+        userConfigurationDataModel.setStorageId(config.getStorageResourceId());
+        userConfigurationDataModel.setExperimentDataDir(config.getUserId()
+                .concat(File.separator)
+                .concat(config.getProjectId())
+                .concat(File.separator)
+                .concat(experimentName));
+
+        experimentModel.setUserConfigurationData(userConfigurationDataModel);
+
+        Airavata.Client airavataClient = AiravataClientFactory.createAiravataSecureClient(apiHost,
apiPort, trustStorePath, trustStorePassword, 100000);
+
+        List<InputDataObjectType> applicationInputs = airavataClient.getApplicationInputs(new
AuthzToken(""),
+                config.getApplicationInterfaceId());
+        List<InputDataObjectType> experimentInputs = new ArrayList<>();
+
+        storageResourceManager.createExperimentDirectory(config.getUserId(), config.getProjectId(),
experimentName);
+
+        for (InputDataObjectType inputDataObjectType: applicationInputs) {
+
+            Optional<Configuration.Input> input = config.getInputs().stream().filter(inp
-> inp.getName().equals(inputDataObjectType.getName())).findFirst();
+
+            if (input.isPresent()) {
+                if (inputDataObjectType.getType() == DataType.URI) {
+                    String localFilePath = input.get().getValue();
+                    String uploadedPath = storageResourceManager.uploadInputFile(airavataClient,
localFilePath, config.getUserId(), config.getProjectId(), experimentName, config.getGatewayId());
+                    inputDataObjectType.setValue(uploadedPath);
+
+                } else if (inputDataObjectType.getType() == DataType.STRING) {
+                    inputDataObjectType.setValue(input.get().getValue());
+                }
+            }
+            experimentInputs.add(inputDataObjectType);
+        }
+
+        experimentModel.setExperimentInputs(experimentInputs);
+        experimentModel.setExperimentOutputs(airavataClient.getApplicationOutputs(new AuthzToken(""),
config.getApplicationInterfaceId()));
+        experimentModel.setExperimentType(ExperimentType.SINGLE_APPLICATION);
+
+        String experimentId = airavataClient.createExperiment(new AuthzToken(""), config.getGatewayId(),
experimentModel);
+
+        airavataClient.launchExperiment(new AuthzToken(""), experimentId, config.getGatewayId());
+        System.out.println(experimentId);
+
+    }
+}
diff --git a/tools/load-client/src/main/resources/bin/load-client.sh b/tools/load-client/src/main/resources/bin/load-client.sh
new file mode 100644
index 0000000..fe05e8d
--- /dev/null
+++ b/tools/load-client/src/main/resources/bin/load-client.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dairavata.config.dir=${AIRAVATA_HOME}/conf -Dairavata.home=${AIRAVATA_HOME}"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+    case ${var} in
+        -xdebug)
+        	AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+            JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=8000"
+            shift
+	    ;;
+        -log)
+            shift
+            LOG_FILE="$1"
+            shift
+            # If relative path, expand to absolute path using the user's $CWD
+            if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+                LOG_FILE="${CWD}/${LOG_FILE}"
+            fi
+        ;;
+        -h)
+            echo "Usage: load-client.sh"
+
+            echo "command options:"
+            echo "  -config             Load configuration file in yml format"
+            echo "  -apiHost            API Server host name"
+            echo "  -apiPort            API Server port"
+            echo "  -privateKeyPath     SSH private key path to communicate with storage
resources (Defaults to user private key in ~/.ssh/id_rsa)"
+            echo "  -publicKeyPath      SSH public key path to communicate with storage resources
(Defaults to user public key in ~/.ssh/id_rsa.pub)"
+            echo "  -passPhrase         SSH private key pass phrase (if any)"
+            echo "  -h                  Display this help and exit"
+            shift
+            exit 0
+        ;;
+	    *)
+	        EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+            shift
+        ;;
+    esac
+done
+
+echo $*
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+    org.apache.airavata.tools.load.LoadClient ${AIRAVATA_COMMAND} ${EXTRA_ARGS} $*
diff --git a/tools/load-client/src/main/resources/bin/setenv.sh b/tools/load-client/src/main/resources/bin/setenv.sh
new file mode 100644
index 0000000..9e894e1
--- /dev/null
+++ b/tools/load-client/src/main/resources/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+  ls=`ls -ld "$PRG"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '.*/.*' > /dev/null; then
+    PRG="$link"
+  else
+    PRG=`dirname "$PRG"`/"$link"
+  fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+  AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
diff --git a/tools/load-client/src/main/resources/client_truststore.jks b/tools/load-client/src/main/resources/client_truststore.jks
new file mode 100644
index 0000000..4ff588f
Binary files /dev/null and b/tools/load-client/src/main/resources/client_truststore.jks differ
diff --git a/tools/load-client/src/main/resources/conf/load-config.yml b/tools/load-client/src/main/resources/conf/load-config.yml
new file mode 100644
index 0000000..a193c36
--- /dev/null
+++ b/tools/load-client/src/main/resources/conf/load-config.yml
@@ -0,0 +1,22 @@
+configurations:
+  - experimentBaseName: "LoadTest"
+    userId: "dimuthu"
+    gatewayId: "seagrid"
+    projectId: "DefaultProject_7ac38275-0ca1-433a-ab6a-630c8c1df2ef"
+    applicationInterfaceId: "Echo_3f480d1f-ea86-4018-94bb-015423d66a1c"
+    computeResourceId: "js-156-93.jetstream-cloud.org_696c097d-a138-4445-b254-cd7e55c84fad"
+    storageResourceId: "testing.seagrid.org_20dbec4c-223e-4568-a267-63d7efc6267e"
+
+    inputs:
+      - name: "Input-to-Echo"
+        value: "Test"
+
+    queue: "cloud"
+    wallTime: 60
+    cpuCount: 2
+    nodeCount: 1
+    physicalMemory: 512
+
+    concurrentUsers: 10
+    iterationsPerUser: 2
+    randomMSDelayWithinSubmissions: 100
diff --git a/tools/phoebus-integration/pom.xml b/tools/phoebus-integration/pom.xml
index 90c9832..95adc10 100644
--- a/tools/phoebus-integration/pom.xml
+++ b/tools/phoebus-integration/pom.xml
@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.airavata</groupId>
         <artifactId>airavata-tools-parent</artifactId>
-        <version>0.12-SNAPSHOT</version>
+        <version>0.17-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 4a46c29..2223efb 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -42,9 +42,8 @@
                 <activeByDefault>true</activeByDefault>
             </activation>
             <modules>
-		<module>registry-tool</module>
-		<module>gsissh</module>
-        </modules>
+                <module>load-client</module>
+            </modules>
         </profile>
     </profiles>
     <!--<properties>-->


Mime
View raw message