Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9DF44174D3 for ; Tue, 7 Apr 2015 16:34:09 +0000 (UTC) Received: (qmail 6976 invoked by uid 500); 7 Apr 2015 16:34:09 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 6935 invoked by uid 500); 7 Apr 2015 16:34:09 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 6926 invoked by uid 99); 7 Apr 2015 16:34:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Apr 2015 16:34:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5A3C4E17A7; Tue, 7 Apr 2015 16:34:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shameera@apache.org To: commits@airavata.apache.org Message-Id: <1b06cda90c1b491783bdc8293075b618@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: airavata git commit: Added Lonestar email parser and refactored parser code Date: Tue, 7 Apr 2015 16:34:09 +0000 (UTC) Repository: airavata Updated Branches: refs/heads/emailMonitoring b110ff250 -> 9f1fa2d35 Added Lonestar email parser and refactored parser code Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/9f1fa2d3 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/9f1fa2d3 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/9f1fa2d3 Branch: refs/heads/emailMonitoring Commit: 9f1fa2d35ebe35652a40db9d04c5e96e77b5eb1c Parents: b110ff2 Author: shamrath Authored: Tue Apr 7 12:34:54 2015 -0400 Committer: shamrath Committed: Tue Apr 7 12:34:54 2015 -0400 ---------------------------------------------------------------------- .../airavata/gfac/core/cpi/BetterGfacImpl.java | 1 - .../core/monitor/mail/EmailBasedMonitor.java | 1 + .../core/monitor/mail/parser/EmailParser.java | 3 ++ .../mail/parser/LonestarEmailParser.java | 52 +++++++++++++++++++- .../monitor/mail/parser/PBSEmailParser.java | 8 ++- .../monitor/mail/parser/SLURMEmailParser.java | 4 +- .../gfac/ssh/provider/impl/SSHProvider.java | 50 ++++++++++--------- 7 files changed, 88 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java index 1d9008b..8d09a09 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java @@ -305,7 +305,6 @@ public class BetterGfacImpl implements GFac,Watcher { jobExecutionContext.setGfac(this); jobExecutionContext.setZk(zk); jobExecutionContext.setCredentialStoreToken(AiravataZKUtils.getExpTokenId(zk, experimentID, taskID)); - jobExecutionContext.setProperty("MonitorPublisher", monitorPublisher); // handle job submission protocol List jobSubmissionInterfaces = computeResource.getJobSubmissionInterfaces(); http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java index 8adb860..f15e582 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java @@ -102,6 +102,7 @@ public class EmailBasedMonitor implements Runnable{ } public void addToJobMonitorMap(String jobId, JobExecutionContext jobExecutionContext) { + log.info("Added Job Id : " + jobId + " to email based monitor map"); jobMonitorMap.put(jobId, jobExecutionContext); } http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java index d35e6b7..ad8e4a3 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java @@ -27,6 +27,9 @@ import javax.mail.Message; import javax.mail.MessagingException; public interface EmailParser { + static final String STATUS = "status"; + static final String JOBID = "jobId"; + static final String EXIT_STATUS = "exitStatus"; JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException; } http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java index 6adde4e..59651da 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java @@ -20,5 +20,55 @@ */ package org.apache.airavata.gfac.core.monitor.mail.parser; -public class LonestarEmailParser { +import org.apache.airavata.common.exception.AiravataException; +import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult; +import org.apache.airavata.model.workspace.experiment.JobState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.mail.Message; +import javax.mail.MessagingException; +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class LonestarEmailParser implements EmailParser { + private static final Logger log = LoggerFactory.getLogger(LonestarEmailParser.class); + //root@c312-206.ls4.tacc.utexas.edu + private static final String SIGNAL = "signal"; + private static final String LONESTAR_REGEX = "Job (?<" + JOBID + ">\\d+) \\(.*\\) (?<" + STATUS + + ">.*)\\s[a-zA-Z =]+(?<" + EXIT_STATUS + ">\\d+)\\sSignal[ ]*=[ ]*(?<" + SIGNAL + ">[a-zA-z]*)"; + + @Override + public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException { + JobStatusResult jobStatusResult = new JobStatusResult(); + try { + String content = ((String) message.getContent()); + Pattern pattern = Pattern.compile(LONESTAR_REGEX); + Matcher matcher = pattern.matcher(content); + if (matcher.find()) { + jobStatusResult.setJobId(matcher.group(JOBID)); + String status = matcher.group(STATUS); + jobStatusResult.setState(getJobState(status, content)); + return jobStatusResult; + } else { + log.error("No matched found for content => \n" + content); + } + } catch (IOException e) { + throw new AiravataException("Error while reading content of the email message"); + } + return jobStatusResult; + } + + private JobState getJobState(String status, String content) { + switch (status) { + case "Aborted": + return JobState.FAILED; + case "Scucess": + return JobState.COMPLETE; + default: + return JobState.UNKNOWN; + } + + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java index 37d2317..cd6142c 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java @@ -36,20 +36,18 @@ public class PBSEmailParser implements EmailParser { private static final Logger log = LoggerFactory.getLogger(PBSEmailParser.class); - private static final String STATUS = "status"; - private static final String JOBID = "jobId"; - private static final String EXIT_STATUS = "exitStatus"; + private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<" + STATUS + ">[a-zA-Z\\ ]*)"; private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)"; @Override public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException { + JobStatusResult jobStatusResult = new JobStatusResult(); try { String content = ((String) message.getContent()); Pattern pattern = Pattern.compile(REGEX); Matcher matcher = pattern.matcher(content); - JobStatusResult jobStatusResult = new JobStatusResult(); if (matcher.find()) { jobStatusResult.setJobId(matcher.group(JOBID)); String statusLine = matcher.group(STATUS); @@ -62,7 +60,7 @@ public class PBSEmailParser implements EmailParser { } catch (IOException e) { throw new AiravataException("Error while reading content of the email message"); } - return null; + return jobStatusResult; } private JobState getJobState(String statusLine, String content) { http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java index e5af721..ceb0013 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java @@ -42,10 +42,10 @@ public class SLURMEmailParser implements EmailParser { @Override public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{ + JobStatusResult jobStatusResult = new JobStatusResult(); String subject = message.getSubject(); Pattern pattern = Pattern.compile(REGEX); Matcher matcher = pattern.matcher(subject); - JobStatusResult jobStatusResult = new JobStatusResult(); if (matcher.find()) { jobStatusResult.setJobId(matcher.group(JOBID)); jobStatusResult.setState(getJobState(matcher.group(STATUS))); @@ -57,7 +57,7 @@ public class SLURMEmailParser implements EmailParser { } else { log.error("No matched found for subject -> " + subject); } - return null; + return jobStatusResult; } private JobState getJobState(String state) { http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java index 0012ce1..32be550 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java @@ -22,12 +22,15 @@ package org.apache.airavata.gfac.ssh.provider.impl; import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.MonitorPublisher; +import org.apache.airavata.common.utils.ServerSettings; import org.apache.airavata.gfac.Constants; +import org.apache.airavata.gfac.ExecutionMode; import org.apache.airavata.gfac.GFacException; import org.apache.airavata.gfac.core.context.JobExecutionContext; import org.apache.airavata.gfac.core.context.MessageContext; +import org.apache.airavata.gfac.core.cpi.BetterGfacImpl; import org.apache.airavata.gfac.core.handler.GFacHandlerException; +import org.apache.airavata.gfac.core.handler.ThreadedHandler; import org.apache.airavata.gfac.core.notification.events.StartExecutionEvent; import org.apache.airavata.gfac.core.provider.AbstractProvider; import org.apache.airavata.gfac.core.provider.GFacProviderException; @@ -367,28 +370,31 @@ public class SSHProvider extends AbstractProvider { } public void delegateToMonitorHandlers(JobExecutionContext jobExecutionContext) throws GFacHandlerException { - /*List daemonHandlers = BetterGfacImpl.getDaemonHandlers(); - if (daemonHandlers == null) { - daemonHandlers = BetterGfacImpl.getDaemonHandlers(); - } - ThreadedHandler pullMonitorHandler = null; - ThreadedHandler pushMonitorHandler = null; - for (ThreadedHandler threadedHandler : daemonHandlers) { - if ("org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler".equals(threadedHandler.getClass().getName())) { - pullMonitorHandler = threadedHandler; - pullMonitorHandler.invoke(jobExecutionContext); + if (ServerSettings.isEmailBasedNotificationEnable()) { + try { + EmailBasedMonitor emailBasedMonitor = EmailBasedMonitor.getInstant(BetterGfacImpl.getMonitorPublisher()); + emailBasedMonitor.addToJobMonitorMap(jobExecutionContext); + } catch (ApplicationSettingsException e) { + throw new GFacHandlerException("Error while delegating job execution context to email based monitor"); + } + } else { + List daemonHandlers = BetterGfacImpl.getDaemonHandlers(); + if (daemonHandlers == null) { + daemonHandlers = BetterGfacImpl.getDaemonHandlers(); + } + ThreadedHandler pullMonitorHandler = null; + ThreadedHandler pushMonitorHandler = null; + for (ThreadedHandler threadedHandler : daemonHandlers) { + if ("org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler".equals(threadedHandler.getClass().getName())) { + pullMonitorHandler = threadedHandler; + pullMonitorHandler.invoke(jobExecutionContext); + } + // have to handle the GridPushMonitorHandler logic + } + if (pullMonitorHandler == null && pushMonitorHandler == null && ExecutionMode.ASYNCHRONOUS.equals(jobExecutionContext.getGFacConfiguration().getExecutionMode())) { + log.error("No Daemon handler is configured in gfac-config.xml, either pull or push, so monitoring will not invoked" + + ", execution is configured as asynchronous, so Outhandler will not be invoked"); } - // have to handle the GridPushMonitorHandler logic - } - if (pullMonitorHandler == null && pushMonitorHandler == null && ExecutionMode.ASYNCHRONOUS.equals(jobExecutionContext.getGFacConfiguration().getExecutionMode())) { - log.error("No Daemon handler is configured in gfac-config.xml, either pull or push, so monitoring will not invoked" + - ", execution is configured as asynchronous, so Outhandler will not be invoked"); - }*/ - try { - EmailBasedMonitor emailBasedMonitor = EmailBasedMonitor.getInstant(((MonitorPublisher) jobExecutionContext.getProperty("MonitorPublisher"))); - emailBasedMonitor.addToJobMonitorMap(jobExecutionContext); - } catch (ApplicationSettingsException e) { - throw new GFacHandlerException("Error while delegating job execution context to email based monitor"); } }