From commits-return-7111-apmail-airavata-commits-archive=airavata.apache.org@airavata.apache.org Fri Sep 26 15:19:48 2014 Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 13E81177FA for ; Fri, 26 Sep 2014 15:19:48 +0000 (UTC) Received: (qmail 50500 invoked by uid 500); 26 Sep 2014 15:19:48 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 50396 invoked by uid 500); 26 Sep 2014 15:19:47 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 49944 invoked by uid 99); 26 Sep 2014 15:19:47 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Sep 2014 15:19:47 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 847269A7810; Fri, 26 Sep 2014 15:19:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shameera@apache.org To: commits@airavata.apache.org Date: Fri, 26 Sep 2014 15:20:01 -0000 Message-Id: <8586092cc299413d8dcdf3df567621d3@git.apache.org> In-Reply-To: <2d6326ad204e4b0791f95b3e2010e0d0@git.apache.org> References: <2d6326ad204e4b0791f95b3e2010e0d0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [15/15] git commit: Merge branch 'master' into orchestratorJobCountImprovment Merge branch 'master' into orchestratorJobCountImprovment Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/eed05a43 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/eed05a43 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/eed05a43 Branch: refs/heads/orchestratorJobThrottleFeature Commit: eed05a43423ded617483946fb7869bd9edd35465 Parents: 50e5cee 72f3524 Author: shamrath Authored: Fri Sep 26 11:14:28 2014 -0400 Committer: shamrath Committed: Fri Sep 26 11:14:28 2014 -0400 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 51 +++++------ .../server/src/main/resources/PBSTemplate.xslt | 3 +- .../server/src/main/resources/SGETemplate.xslt | 6 +- .../src/main/resources/SLURMTemplate.xslt | 5 +- .../airavata/gfac/server/GfacServerHandler.java | 25 +++-- .../airavata/gfac/core/cpi/BetterGfacImpl.java | 19 +--- .../gfac/core/utils/GFacThreadPoolExecutor.java | 35 +++++++ .../gfac/core/utils/InputHandlerWorker.java | 51 +++++++++++ .../handler/GSISSHDirectorySetupHandler.java | 29 +++--- .../gfac/gsissh/handler/GSISSHInputHandler.java | 5 + .../gsissh/handler/GSISSHOutputHandler.java | 34 +++---- .../gsissh/provider/impl/GSISSHProvider.java | 7 -- .../gfac/gsissh/util/GFACGSISSHUtils.java | 1 + .../monitor/impl/pull/qstat/HPCPullMonitor.java | 96 ++++++++++---------- .../impl/push/amqp/SimpleJobFinishConsumer.java | 8 +- .../ssh/handler/AdvancedSCPInputHandler.java | 12 +++ .../ssh/handler/AdvancedSCPOutputHandler.java | 23 ++++- .../ssh/handler/SSHDirectorySetupHandler.java | 14 +-- .../gfac/ssh/handler/SSHInputHandler.java | 5 + .../gfac/ssh/handler/SSHOutputHandler.java | 28 +++--- .../airavata/gfac/ssh/util/GFACSSHUtils.java | 1 + .../airavata/gsi/ssh/api/job/JobDescriptor.java | 10 ++ .../gsi/ssh/api/job/SGEOutputParser.java | 16 ++-- .../apache/airavata/gsi/ssh/impl/JobStatus.java | 4 +- .../main/resources/schemas/PBSJobDescriptor.xsd | 1 + 25 files changed, 319 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java ---------------------------------------------------------------------- diff --cc modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java index 5f83b43,b588894..4399d06 --- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java +++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java @@@ -264,29 -261,22 +263,32 @@@ public class HPCPullMonitor extends Pul // they become empty Map jobRemoveCountMap = new HashMap(); ZooKeeper zk = null; - for (MonitorID completedJob : completedJobs) { + Set keys = completedJobs.keySet(); + for (String jobName: keys) { + MonitorID completedJob = completedJobs.get(jobName); + GFacThreadPoolExecutor.getCachedThreadPool().submit(new OutHandlerWorker(gfac, completedJob, publisher)); CommonUtils.removeMonitorFromQueue(queue, completedJob); - if (zk == null) { - zk = completedJob.getJobExecutionContext().getZk(); - } - String key = CommonUtils.getJobCountUpdatePath(completedJob); - int i = 0; - if (jobRemoveCountMap.containsKey(key)) { - i = Integer.valueOf(jobRemoveCountMap.get(key)); + if (ServerSettings.getEnableMaxJobCountCheck().equals("true")) { // is job restriction available? + TaskDetails taskDetails = completedJob.getJobExecutionContext().getTaskData(); + ComputeResourceDescription computeResourceDesc = CommonUtils.getComputeResourceDescription( + taskDetails); + if (computeResourceDesc.getBatchQueuesSize() > 0 && + taskDetails.getTaskScheduling().getQueueName() != null) { + if (zk == null) { + zk = completedJob.getJobExecutionContext().getZk(); + } + String key = CommonUtils.getJobCountUpdatePath(completedJob); + int i = 0; + if (jobRemoveCountMap.containsKey(key)) { + i = Integer.valueOf(jobRemoveCountMap.get(key)); + } + jobRemoveCountMap.put(key, ++i); + } else { + // ignore + } } - jobRemoveCountMap.put(key, ++i); } - if (completedJobs.size() > 0) { + if (ServerSettings.getEnableMaxJobCountCheck().equals("true") && completedJobs.size() > 0) { // reduce completed job count from zookeeper CommonUtils.updateZkWithJobCount(zk, jobRemoveCountMap, false); } http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPInputHandler.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java ----------------------------------------------------------------------