Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 589EC10F37 for ; Wed, 19 Feb 2014 20:42:28 +0000 (UTC) Received: (qmail 84638 invoked by uid 500); 19 Feb 2014 20:42:26 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 84559 invoked by uid 500); 19 Feb 2014 20:42:24 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 84529 invoked by uid 99); 19 Feb 2014 20:42:24 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Feb 2014 20:42:24 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A663C928398; Wed, 19 Feb 2014 20:42:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: lahiru@apache.org To: commits@airavata.apache.org Date: Wed, 19 Feb 2014 20:42:23 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: Adding basic structure to monitoring component - AIRAVATA-1021 Repository: airavata Updated Branches: refs/heads/master a6200bbc1 -> f0f290652 Adding basic structure to monitoring component - AIRAVATA-1021 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/e2647167 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/e2647167 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/e2647167 Branch: refs/heads/master Commit: e2647167d803b5b4c0dc5eaba5a478944ab3a4fe Parents: af9cd3b Author: lahiru Authored: Wed Feb 19 15:42:01 2014 -0500 Committer: lahiru Committed: Wed Feb 19 15:42:01 2014 -0500 ---------------------------------------------------------------------- modules/airavata-job-monitor/pom.xml | 289 +++++++++++++++++++ .../job/monitor/AiravataJobStatusUpdator.java | 76 +++++ .../apache/airavata/job/monitor/Monitor.java | 28 ++ .../apache/airavata/job/monitor/MonitorID.java | 64 ++++ .../airavata/job/monitor/MonitorManager.java | 84 ++++++ .../airavata/job/monitor/PullMonitor.java | 30 ++ .../airavata/job/monitor/PushMonitor.java | 30 ++ .../monitor/core/AiravataAbstractMonitor.java | 46 +++ .../airavata/job/monitor/core/Monitor.java | 31 ++ .../airavata/job/monitor/core/PullMonitor.java | 64 ++++ .../airavata/job/monitor/core/PushMonitor.java | 52 ++++ .../job/monitor/event/MonitorPublisher.java | 43 +++ .../exception/AiravataMonitorException.java | 37 +++ .../monitor/impl/pull/qstat/QstatMonitor.java | 53 ++++ .../job/monitor/impl/push/amqp/AMQPMonitor.java | 46 +++ .../monitor/impl/push/amqp/BasicConsumer.java | 25 ++ .../airavata/job/monitor/state/JobStatus.java | 52 ++++ .../job/monitor/state/JobStatusInfo.java | 48 +++ .../monitor/state/impl/AmazonJobStatusInfo.java | 39 +++ .../monitor/state/impl/GridJobStatusInfo.java | 40 +++ 20 files changed, 1177 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/pom.xml ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/pom.xml b/modules/airavata-job-monitor/pom.xml new file mode 100644 index 0000000..ca86218 --- /dev/null +++ b/modules/airavata-job-monitor/pom.xml @@ -0,0 +1,289 @@ + + + + + + + org.apache.airavata + airavata + 0.12-SNAPSHOT + ../../pom.xml + + + 4.0.0 + airavata-gfac-core + Airavata Job Monitor + This component handle the Airavata Job monitoring funcationality + http://airavata.apache.org/ + + + + + org.jglobus + gss + ${jglobus.version} + + + org.jglobus + gram + ${jglobus.version} + + + org.bouncycastle + bcprov-jdk16 + + + + + org.bouncycastle + bcprov-jdk16 + + + org.jglobus + myproxy + ${jglobus.version} + + + org.jglobus + gridftp + ${jglobus.version} + + + + + org.slf4j + slf4j-api + + + + + org.apache.airavata + airavata-client-api + ${project.version} + + + org.apache.airavata + airavata-workflow-execution-context + ${project.version} + + + org.apache.airavata + airavata-registry-api + ${project.version} + + + org.apache.airavata + airavata-registry-cpi + ${project.version} + + + + org.apache.airavata + airavata-workflow-tracking + ${project.version} + + + + + net.schmizz + sshj + 0.8.0 + + + + + org.apache.airavata + airavata-credential-store + ${project.version} + + + + + com.amazonaws + aws-java-sdk + 1.3.20 + + + sshtools + j2ssh-core + 0.2.9 + + + sshtools + j2ssh-common + 0.2.9 + + + org.apache.httpcomponents + httpclient + 4.3 + jar + + + org.apache.httpcomponents + httpcore + 4.3 + jar + + + + + junit + junit + test + + + org.testng + testng + 6.1.1 + test + + + org.slf4j + jcl-over-slf4j + test + + + org.slf4j + slf4j-log4j12 + test + + + + eu.unicore + ogsabes-client + 1.7.0-rc + + + org.apache.santuario + xmlsec + + + org.bouncycastle + bcprov-jdk16 + + + + + + + + org.apache.hadoop + hadoop-core + 1.0.3 + + + org.apache.hadoop + hadoop-client + 1.0.3 + + + org.apache.whirr + whirr-core + 0.7.1 + + + org.apache.whirr + whirr-hadoop + 0.7.1 + + + org.hamcrest + hamcrest-all + 1.1 + + + org.mockito + mockito-all + 1.8.5 + + + commons-configuration + commons-configuration + 1.7 + + + net.sf.jopt-simple + jopt-simple + 3.2 + + + + com.google.guava + guava + 12.0 + + + + + org.apache.airavata + gsissh + ${project.version} + + + org.apache.airavata + airavata-data-models + ${project.version} + + + com.jcraft + jsch + 0.1.50 + + + org.ogce + bcgss + 146 + + + org.apache.xmlbeans + xmlbeans + ${xmlbeans.version} + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/services/** + **/gfac/** + + always + false + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + target/lib + false + true + + + + + + + + http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java new file mode 100644 index 0000000..1009565 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/AiravataJobStatusUpdator.java @@ -0,0 +1,76 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor; + +import com.google.common.eventbus.Subscribe; +import org.apache.airavata.job.monitor.state.JobStatus; +import org.apache.airavata.model.experiment.JobState; +import org.apache.airavata.registry.cpi.Registry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.BlockingQueue; + +public class AiravataJobStatusUpdator{ + private final static Logger logger = LoggerFactory.getLogger(AiravataJobStatusUpdator.class); + + private Registry airavataRegistry; + + private BlockingQueue jobsToMonitor; + + + public AiravataJobStatusUpdator(Registry airavataRegistry, BlockingQueue jobsToMonitor) { + this.airavataRegistry = airavataRegistry; + this.jobsToMonitor = jobsToMonitor; + } + + public Registry getAiravataRegistry() { + return airavataRegistry; + } + + public void setAiravataRegistry(Registry airavataRegistry) { + this.airavataRegistry = airavataRegistry; + } + + public BlockingQueue getJobsToMonitor() { + return jobsToMonitor; + } + + public void setJobsToMonitor(BlockingQueue jobsToMonitor) { + this.jobsToMonitor = jobsToMonitor; + } + + @Subscribe + public void updateRegistry(JobStatus jobStatus) { + /* Here we need to parse the jobStatus message and update + the registry accordingly, for now we are just printing to standard Out + */ + JobState state = jobStatus.getState(); + System.out.println("Job ID: " + jobStatus.getMonitorID().getJobID()); + System.out.println("Username: " + jobStatus.getMonitorID().getUserName()); + System.out.println("Job Status: " + jobStatus.getState().toString()); + if (JobState.COMPLETE.equals(jobStatus.getState())) { + // When job is done we remove the job from the queue + logger.info("Job ID:" + jobStatus.getMonitorID().getJobID() + "is DONE"); + jobsToMonitor.remove(jobStatus.getMonitorID()); + } + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/Monitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/Monitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/Monitor.java new file mode 100644 index 0000000..4712522 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/Monitor.java @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor; + +/** + * This is the primary interface for Monitors, + * This can be used to implement different methods of monitoring + */ +public interface Monitor { +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorID.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorID.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorID.java new file mode 100644 index 0000000..0318ebe --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorID.java @@ -0,0 +1,64 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Timestamp; + +/* +This is the object which contains the data to identify a particular +Job to start the monitoring +*/ +public class MonitorID { + private final static Logger logger = LoggerFactory.getLogger(MonitorID.class); + + private String userName; + + private String jobID; + + private Timestamp lastMonitored; + + public Timestamp getLastMonitored() { + return lastMonitored; + } + + public void setLastMonitored(Timestamp lastMonitored) { + this.lastMonitored = lastMonitored; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getJobID() { + return jobID; + } + + public void setJobID(String jobID) { + this.jobID = jobID; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorManager.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorManager.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorManager.java new file mode 100644 index 0000000..971f222 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/MonitorManager.java @@ -0,0 +1,84 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor; + +import com.google.common.eventbus.EventBus; +import org.apache.airavata.job.monitor.core.PullMonitor; +import org.apache.airavata.job.monitor.core.PushMonitor; +import org.apache.airavata.job.monitor.event.MonitorPublisher; +import org.apache.airavata.job.monitor.exception.AiravataMonitorException; +import org.apache.airavata.persistance.registry.jpa.impl.RegistryImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; + +/* +this is the manager class for monitoring system of airavata, +This simply handle the monitoring flow of the system. +Keeps available jobs to monitor in a queue and once they are done +remove them from the queue, this will be done by AiravataJobUpdator. + */ +public class MonitorManager { + private final static Logger logger = LoggerFactory.getLogger(MonitorManager.class); + + List pullMonitors; + + List pushMonitors; + + BlockingQueue jobsToMonitor; + + MonitorPublisher monitorPublisher; + + /** + * This will initialize the major monitoring system. + */ + public MonitorManager() { + pullMonitors = new ArrayList(); + pushMonitors = new ArrayList(); + jobsToMonitor = new LinkedBlockingDeque(); + monitorPublisher = new MonitorPublisher(new EventBus()); + monitorPublisher.registerListener(new AiravataJobStatusUpdator(new RegistryImpl(),jobsToMonitor)); + } + + public void addPushMonitor(PushMonitor monitor) { + pushMonitors.add(monitor); + } + + public void addPullMonitor(PullMonitor monitor) { + pullMonitors.add(monitor); + } + + /** + * In this method we assume that we give higher preference to Push + * Monitorig mechanism if there's any configured, otherwise Pull + * monitoring will be launched. + * Ex: If there's a reasource which doesn't support Push, we have + * to live with Pull MOnitoring. + * @throws AiravataMonitorException + */ + public void launchMonitor() throws AiravataMonitorException{ + + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PullMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PullMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PullMonitor.java new file mode 100644 index 0000000..78bedf6 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PullMonitor.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor; + +/** + * PullMonitors can implement this interface + * Since the pull and push based monitoring required different + * operations, PullMonitor will be useful. + * This will allow users to program Pull monitors separately + */ +public interface PullMonitor { +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PushMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PushMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PushMonitor.java new file mode 100644 index 0000000..8dd4fa7 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/PushMonitor.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor; + +/** + * PushMonitors can implement this interface + * Since the pull and push based monitoring required different + * operations, PullMonitor will be useful. + * This interface will allow users to program Push monitors separately + */ +public interface PushMonitor { +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/AiravataAbstractMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/AiravataAbstractMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/AiravataAbstractMonitor.java new file mode 100644 index 0000000..2e85b32 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/AiravataAbstractMonitor.java @@ -0,0 +1,46 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.core; + +import org.apache.airavata.job.monitor.event.MonitorPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the abstract Monitor which needs to be used by + * any Monitoring implementation which expect to consume + * to store the status to registry. Because they have to + * use the MonitorPublisher to publish the monitoring statuses + * to the Event Bus. All the Monitor statuses publish to the eventbus + * will be saved to the Registry. + */ +public abstract class AiravataAbstractMonitor implements Monitor { + private final static Logger logger = LoggerFactory.getLogger(AiravataAbstractMonitor.class); + MonitorPublisher publisher; + + public MonitorPublisher getPublisher() { + return publisher; + } + + public void setPublisher(MonitorPublisher publisher) { + this.publisher = publisher; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/Monitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/Monitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/Monitor.java new file mode 100644 index 0000000..ce8cf22 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/Monitor.java @@ -0,0 +1,31 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.core; + +import org.apache.airavata.job.monitor.event.MonitorPublisher; + +/** + * This is the primary interface for Monitors, + * This can be used to implement different methods of monitoring + */ +public interface Monitor { + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PullMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PullMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PullMonitor.java new file mode 100644 index 0000000..4b904b1 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PullMonitor.java @@ -0,0 +1,64 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.core; + +import org.apache.airavata.job.monitor.exception.AiravataMonitorException; + +/** + * PullMonitors can implement this interface + * Since the pull and push based monitoring required different + * operations, PullMonitor will be useful. + * This will allow users to program Pull monitors separately + */ +public abstract class PullMonitor extends AiravataAbstractMonitor{ + + private int pollingFrequence; + /** + * This method will can invoke when PullMonitor needs to start + * and it has to invoke in the frequency specified below, + * @return if the start process is successful return true else false + */ + public abstract boolean startPulling() throws AiravataMonitorException; + + /** + * This is the method to stop the polling process + * @return if the stopping process is successful return true else false + */ + public abstract boolean stopPulling()throws AiravataMonitorException; + + /** + * this method can be used to set the polling frequencey or otherwise + * can implement a polling mechanism, and implement how to do + * @param frequence + */ + public void setPollingFrequence(int frequence){ + this.pollingFrequence = frequence; + } + + /** + * this method can be used to get the polling frequencey or otherwise + * can implement a polling mechanism, and implement how to do + * @return + */ + public int getPollingFrequence(){ + return this.pollingFrequence; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PushMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PushMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PushMonitor.java new file mode 100644 index 0000000..01793ac --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/core/PushMonitor.java @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.core; + +import org.apache.airavata.job.monitor.MonitorID; +import org.apache.airavata.job.monitor.exception.AiravataMonitorException; + +/** + * PushMonitors can implement this interface + * Since the pull and push based monitoring required different + * operations, PullMonitor will be useful. + * This interface will allow users to program Push monitors separately + */ +public abstract class PushMonitor extends AiravataAbstractMonitor { + /** + * This method can be invoked to register a listener with the + * remote monitoring system, ideally inside this method users will be + * writing some client listener code for the remote monitoring system, + * this will be a simple wrapper around any client for the remote Monitor. + * @param monitorID + * @return + */ + public abstract boolean registerListener(MonitorID monitorID)throws AiravataMonitorException; + + /** + * This method can be invoked to unregister a listener with the + * remote monitoring system, ideally inside this method users will be + * writing some client listener code for the remote monitoring system, + * this will be a simple wrapper around any client for the remote Monitor. + * @param monitorID + * @return + */ + public abstract boolean unRegisterListener(MonitorID monitorID)throws AiravataMonitorException; +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/event/MonitorPublisher.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/event/MonitorPublisher.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/event/MonitorPublisher.java new file mode 100644 index 0000000..12c27fa --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/event/MonitorPublisher.java @@ -0,0 +1,43 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.event; + +import com.google.common.eventbus.EventBus; +import org.apache.airavata.job.monitor.state.JobStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MonitorPublisher { + private final static Logger logger = LoggerFactory.getLogger(MonitorPublisher.class); + private EventBus eventBus; + + public MonitorPublisher(EventBus eventBus) { + this.eventBus = eventBus; + } + + public void registerListener(Object listener) { + eventBus.register(listener); + } + + public void publish(JobStatus jobState) { + eventBus.post(jobState); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/exception/AiravataMonitorException.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/exception/AiravataMonitorException.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/exception/AiravataMonitorException.java new file mode 100644 index 0000000..5a13be2 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/exception/AiravataMonitorException.java @@ -0,0 +1,37 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.exception; + +public class AiravataMonitorException extends Exception { + private static final long serialVersionUID = -2849422320139467602L; + + public AiravataMonitorException(Throwable e) { + super(e); + } + + public AiravataMonitorException(String message) { + super(message, null); + } + + public AiravataMonitorException(String message, Throwable e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java new file mode 100644 index 0000000..dcab406 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/pull/qstat/QstatMonitor.java @@ -0,0 +1,53 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.impl.pull.qstat; + +import org.apache.airavata.job.monitor.core.PullMonitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This monitor is based on qstat command which can be run + * in grid resources and retrieve the job status. + */ +public class QstatMonitor extends PullMonitor { + private final static Logger logger = LoggerFactory.getLogger(QstatMonitor.class); + /** + * This method will can invoke when PullMonitor needs to start + * and it has to invoke in the frequency specified below, + * @return if the start process is successful return true else false + */ + public boolean startPulling(){ + return true; + } + + /** + * This is the method to stop the polling process + * @return if the stopping process is successful return true else false + */ + public boolean stopPulling(){ + return true; + } + + public boolean authenticate() { + return false; //To change body of implemented methods use File | Settings | File Templates. + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/AMQPMonitor.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/AMQPMonitor.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/AMQPMonitor.java new file mode 100644 index 0000000..3f836b4 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/AMQPMonitor.java @@ -0,0 +1,46 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.impl.push.amqp; + +import org.apache.airavata.job.monitor.MonitorID; +import org.apache.airavata.job.monitor.core.PushMonitor; +import org.apache.airavata.job.monitor.exception.AiravataMonitorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the implementation for AMQP based monitor, this uses + * rabbitmq client to recieve AMQP based monitoring data from + * mostly excede resources. + */ +public class AMQPMonitor extends PushMonitor { + private final static Logger logger = LoggerFactory.getLogger(AMQPMonitor.class); + + @Override + public boolean registerListener(MonitorID monitorID) throws AiravataMonitorException{ + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public boolean unRegisterListener(MonitorID monitorID) throws AiravataMonitorException{ + return false; //To change body of implemented methods use File | Settings | File Templates. + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/BasicConsumer.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/BasicConsumer.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/BasicConsumer.java new file mode 100644 index 0000000..2399271 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/impl/push/amqp/BasicConsumer.java @@ -0,0 +1,25 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.impl.push.amqp; + +public class BasicConsumer { + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatus.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatus.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatus.java new file mode 100644 index 0000000..7cf2e38 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatus.java @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.state; + +import org.apache.airavata.job.monitor.MonitorID; + +/** + * This is the primary job state object used in + * through out the monitor module. This use airavata-data-model JobState enum + * Ideally after processing each event or monitoring message from remote system + * Each monitoring implementation has to return this object with a state and + * the monitoring ID + */ +public class JobStatus { + private org.apache.airavata.model.experiment.JobState state; + + private MonitorID monitorID; + + public org.apache.airavata.model.experiment.JobState getState() { + return state; + } + + public void setState(org.apache.airavata.model.experiment.JobState state) { + this.state = state; + } + + public MonitorID getMonitorID() { + return monitorID; + } + + public void setMonitorID(MonitorID monitorID) { + this.monitorID = monitorID; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatusInfo.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatusInfo.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatusInfo.java new file mode 100644 index 0000000..10d5ca2 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/JobStatusInfo.java @@ -0,0 +1,48 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.state; + +import org.apache.airavata.gsi.ssh.impl.JobStatus; + +/** + * Based on the job status monitoring we can gather + * different informaation about the job, its not simply + * the job status, so we need a way to implement + * different job statusinfo object to keep job status + */ +public interface JobStatusInfo { + + /** + * This method can be used to get JobStatusInfo data and + * decide the finalJobState + * + * @param jobState + */ + void setJobStatus(JobStatus jobState); + + /** + * After setting the jobState by processing jobinformation + * this method can be used to get the JobStatus + * @return + */ + JobStatus getJobStatus(); + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/AmazonJobStatusInfo.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/AmazonJobStatusInfo.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/AmazonJobStatusInfo.java new file mode 100644 index 0000000..385c430 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/AmazonJobStatusInfo.java @@ -0,0 +1,39 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.state.impl; + +import org.apache.airavata.gsi.ssh.impl.JobStatus; +import org.apache.airavata.job.monitor.state.JobStatusInfo; + +/** + * This can be used to store job status information about + * amazon jobs, this data could be very different from + * a typical grid job + */ +public class AmazonJobStatusInfo implements JobStatusInfo { + public void setJobStatus(JobStatus jobState) { + //To change body of implemented methods use File | Settings | File Templates. + } + + public JobStatus getJobStatus() { + return null; //To change body of implemented methods use File | Settings | File Templates. + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/e2647167/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/GridJobStatusInfo.java ---------------------------------------------------------------------- diff --git a/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/GridJobStatusInfo.java b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/GridJobStatusInfo.java new file mode 100644 index 0000000..3c3b421 --- /dev/null +++ b/modules/airavata-job-monitor/src/main/java/org/apache/airavata/job/monitor/state/impl/GridJobStatusInfo.java @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.job.monitor.state.impl; + +import org.apache.airavata.gsi.ssh.impl.JobStatus; +import org.apache.airavata.job.monitor.state.JobStatusInfo; + + +/** + * This can be used to keep information about a Grid job + * which we can get from qstat polling or from amqp based + * monitoring in Grid machines + */ +public class GridJobStatusInfo implements JobStatusInfo { + public void setJobStatus(JobStatus jobState) { + //To change body of implemented methods use File | Settings | File Templates. + } + + public JobStatus getJobStatus() { + return null; //To change body of implemented methods use File | Settings | File Templates. + } +}