airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shame...@apache.org
Subject [15/15] git commit: Merge branch 'master' into orchestratorJobCountImprovment
Date Fri, 26 Sep 2014 15:20:01 GMT
Merge branch 'master' into orchestratorJobCountImprovment


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/eed05a43
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/eed05a43
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/eed05a43

Branch: refs/heads/orchestratorJobThrottleFeature
Commit: eed05a43423ded617483946fb7869bd9edd35465
Parents: 50e5cee 72f3524
Author: shamrath <shameerainfo@gmail.com>
Authored: Fri Sep 26 11:14:28 2014 -0400
Committer: shamrath <shameerainfo@gmail.com>
Committed: Fri Sep 26 11:14:28 2014 -0400

----------------------------------------------------------------------
 .../client/samples/CreateLaunchExperiment.java  | 51 +++++------
 .../server/src/main/resources/PBSTemplate.xslt  |  3 +-
 .../server/src/main/resources/SGETemplate.xslt  |  6 +-
 .../src/main/resources/SLURMTemplate.xslt       |  5 +-
 .../airavata/gfac/server/GfacServerHandler.java | 25 +++--
 .../airavata/gfac/core/cpi/BetterGfacImpl.java  | 19 +---
 .../gfac/core/utils/GFacThreadPoolExecutor.java | 35 +++++++
 .../gfac/core/utils/InputHandlerWorker.java     | 51 +++++++++++
 .../handler/GSISSHDirectorySetupHandler.java    | 29 +++---
 .../gfac/gsissh/handler/GSISSHInputHandler.java |  5 +
 .../gsissh/handler/GSISSHOutputHandler.java     | 34 +++----
 .../gsissh/provider/impl/GSISSHProvider.java    |  7 --
 .../gfac/gsissh/util/GFACGSISSHUtils.java       |  1 +
 .../monitor/impl/pull/qstat/HPCPullMonitor.java | 96 ++++++++++----------
 .../impl/push/amqp/SimpleJobFinishConsumer.java |  8 +-
 .../ssh/handler/AdvancedSCPInputHandler.java    | 12 +++
 .../ssh/handler/AdvancedSCPOutputHandler.java   | 23 ++++-
 .../ssh/handler/SSHDirectorySetupHandler.java   | 14 +--
 .../gfac/ssh/handler/SSHInputHandler.java       |  5 +
 .../gfac/ssh/handler/SSHOutputHandler.java      | 28 +++---
 .../airavata/gfac/ssh/util/GFACSSHUtils.java    |  1 +
 .../airavata/gsi/ssh/api/job/JobDescriptor.java | 10 ++
 .../gsi/ssh/api/job/SGEOutputParser.java        | 16 ++--
 .../apache/airavata/gsi/ssh/impl/JobStatus.java |  4 +-
 .../main/resources/schemas/PBSJobDescriptor.xsd |  1 +
 25 files changed, 319 insertions(+), 170 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --cc modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
index 5f83b43,b588894..4399d06
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
@@@ -264,29 -261,22 +263,32 @@@ public class HPCPullMonitor extends Pul
              // they become empty
              Map<String, Integer> jobRemoveCountMap = new HashMap<String, Integer>();
              ZooKeeper zk = null;
-             for (MonitorID completedJob : completedJobs) {
+             Set<String> keys = completedJobs.keySet();
+             for (String jobName: keys) {
+                 MonitorID completedJob = completedJobs.get(jobName);
+                 GFacThreadPoolExecutor.getCachedThreadPool().submit(new OutHandlerWorker(gfac,
completedJob, publisher));
                  CommonUtils.removeMonitorFromQueue(queue, completedJob);
 -                if (zk == null) {
 -                    zk = completedJob.getJobExecutionContext().getZk();
 -                }
 -                String key = CommonUtils.getJobCountUpdatePath(completedJob);
 -                int i = 0;
 -                if (jobRemoveCountMap.containsKey(key)) {
 -                    i = Integer.valueOf(jobRemoveCountMap.get(key));
 +                if (ServerSettings.getEnableMaxJobCountCheck().equals("true")) { // is job
restriction available?
 +                    TaskDetails taskDetails = completedJob.getJobExecutionContext().getTaskData();
 +                    ComputeResourceDescription computeResourceDesc = CommonUtils.getComputeResourceDescription(
 +                            taskDetails);
 +                    if (computeResourceDesc.getBatchQueuesSize() > 0 &&
 +                            taskDetails.getTaskScheduling().getQueueName() != null) {
 +                        if (zk == null) {
 +                            zk = completedJob.getJobExecutionContext().getZk();
 +                        }
 +                        String key = CommonUtils.getJobCountUpdatePath(completedJob);
 +                        int i = 0;
 +                        if (jobRemoveCountMap.containsKey(key)) {
 +                            i = Integer.valueOf(jobRemoveCountMap.get(key));
 +                        }
 +                        jobRemoveCountMap.put(key, ++i);
 +                    } else {
 +                        // ignore
 +                    }
                  }
 -                jobRemoveCountMap.put(key, ++i);
              }
 -            if (completedJobs.size() > 0) {
 +            if (ServerSettings.getEnableMaxJobCountCheck().equals("true") && completedJobs.size()
> 0) {
                  // reduce completed job count from zookeeper
                  CommonUtils.updateZkWithJobCount(zk, jobRemoveCountMap, false);
              }

http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPInputHandler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/airavata/blob/eed05a43/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
----------------------------------------------------------------------


Mime
View raw message