Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B28D611FEB for ; Wed, 21 May 2014 17:56:19 +0000 (UTC) Received: (qmail 13485 invoked by uid 500); 21 May 2014 17:56:19 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 13444 invoked by uid 500); 21 May 2014 17:56:19 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 13332 invoked by uid 99); 21 May 2014 17:56:19 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 May 2014 17:56:19 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 6B5BE99AE6F; Wed, 21 May 2014 17:56:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lahiru@apache.org To: commits@airavata.apache.org Date: Wed, 21 May 2014 17:56:50 -0000 Message-Id: In-Reply-To: <222cd90a259443bdbcb824e89aa2c2b2@git.apache.org> References: <222cd90a259443bdbcb824e89aa2c2b2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [36/51] [abbrv] git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata Conflicts: modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/ee79b70b Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/ee79b70b Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/ee79b70b Branch: refs/heads/temp Commit: ee79b70b6ddc71ac8999ca6ccea240555ce5b967 Parents: 9be225a 14c1841 Author: raminder Authored: Tue May 6 10:37:00 2014 -0400 Committer: raminder Committed: Tue May 6 10:37:00 2014 -0400 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 3 +- .../airavata/common/utils/ServerSettings.java | 5 + .../main/resources/airavata-server.properties | 4 +- .../server/src/main/resources/gfac-config.xml | 47 +- modules/distribution/server/pom.xml | 5 + .../server/src/main/assembly/bin-assembly.xml | 1 + .../gfac/bes/provider/impl/BESProvider.java | 568 ++++++++++++++ .../gfac/bes/security/GSISecurityContext.java | 288 ++++++++ .../gfac/bes/utils/ApplicationProcessor.java | 252 +++++++ .../gfac/bes/utils/DataStagingProcessor.java | 236 ++++++ .../gfac/bes/utils/DataTransferrer.java | 241 ++++++ .../airavata/gfac/bes/utils/FileDownloader.java | 256 +++++++ .../gfac/bes/utils/FileTransferBase.java | 227 ++++++ .../airavata/gfac/bes/utils/FileUploader.java | 245 ++++++ .../airavata/gfac/bes/utils/JSDLGenerator.java | 103 +++ .../airavata/gfac/bes/utils/JSDLUtils.java | 540 ++++++++++++++ .../apache/airavata/gfac/bes/utils/Mode.java | 45 ++ .../airavata/gfac/bes/utils/OSRequirement.java | 108 +++ .../apache/airavata/gfac/bes/utils/OSType.java | 124 ++++ .../gfac/bes/utils/ProcessorRequirement.java | 61 ++ .../airavata/gfac/bes/utils/RangeValueType.java | 274 +++++++ .../gfac/bes/utils/ResourceProcessor.java | 152 ++++ .../airavata/gfac/bes/utils/SPMDProcessor.java | 33 + .../airavata/gfac/bes/utils/SPMDVariations.java | 52 ++ .../airavata/gfac/bes/utils/StorageCreator.java | 211 ++++++ .../gfac/bes/utils/UASDataStagingProcessor.java | 225 ++++++ .../airavata/gfac/bes/utils/URIUtils.java | 119 +++ .../context/security/GSISecurityContext.java | 288 -------- .../gfac/provider/impl/BESProvider.java | 567 -------------- .../gfac/utils/ApplicationProcessor.java | 252 ------- .../gfac/utils/DataStagingProcessor.java | 235 ------ .../airavata/gfac/utils/DataTransferrer.java | 241 ------ .../airavata/gfac/utils/FileDownloader.java | 256 ------- .../airavata/gfac/utils/FileTransferBase.java | 227 ------ .../airavata/gfac/utils/FileUploader.java | 245 ------ .../airavata/gfac/utils/JSDLGenerator.java | 105 --- .../apache/airavata/gfac/utils/JSDLUtils.java | 540 -------------- .../org/apache/airavata/gfac/utils/Mode.java | 45 -- .../airavata/gfac/utils/OSRequirement.java | 108 --- .../org/apache/airavata/gfac/utils/OSType.java | 124 ---- .../gfac/utils/ProcessorRequirement.java | 61 -- .../airavata/gfac/utils/RangeValueType.java | 274 ------- .../airavata/gfac/utils/ResourceProcessor.java | 152 ---- .../airavata/gfac/utils/SPMDProcessor.java | 33 - .../airavata/gfac/utils/SPMDVariations.java | 52 -- .../airavata/gfac/utils/StorageCreator.java | 211 ------ .../gfac/utils/UASDataStagingProcessor.java | 225 ------ .../apache/airavata/gfac/utils/URIUtils.java | 119 --- .../impl/JSDLGeneratorTestWithMyProxyAuth.java | 32 - .../gfac-bes/src/test/resources/gfac-config.xml | 11 +- modules/gfac/gfac-core/pom.xml | 36 - .../org/apache/airavata/gfac/Constants.java | 2 + .../org/apache/airavata/gfac/ExecutionMode.java | 39 + .../apache/airavata/gfac/GFacConfiguration.java | 27 +- .../airavata/gfac/JobSubmissionFault.java | 7 +- .../org/apache/airavata/gfac/Scheduler.java | 50 +- .../airavata/gfac/context/AbstractContext.java | 50 -- .../gfac/context/ApplicationContext.java | 57 -- .../gfac/context/JobExecutionContext.java | 256 ------- .../airavata/gfac/context/MessageContext.java | 51 -- .../gfac/core/context/AbstractContext.java | 50 ++ .../gfac/core/context/ApplicationContext.java | 57 ++ .../gfac/core/context/JobExecutionContext.java | 276 +++++++ .../gfac/core/context/MessageContext.java | 51 ++ .../org/apache/airavata/gfac/core/cpi/GFac.java | 42 ++ .../apache/airavata/gfac/core/cpi/GFacImpl.java | 441 +++++++++++ .../gfac/core/handler/AbstractHandler.java | 60 ++ .../core/handler/AppDescriptorCheckHandler.java | 87 +++ .../airavata/gfac/core/handler/GFacHandler.java | 33 + .../gfac/core/handler/GFacHandlerConfig.java | 51 ++ .../gfac/core/handler/GFacHandlerException.java | 49 ++ .../gfac/core/handler/ThreadedHandler.java | 31 + .../core/monitor/AbstractActivityListener.java | 27 + .../AiravataExperimentStatusUpdator.java | 80 ++ .../core/monitor/AiravataJobStatusUpdator.java | 127 ++++ .../core/monitor/AiravataTaskStatusUpdator.java | 112 +++ .../AiravataWorkflowNodeStatusUpdator.java | 111 +++ .../gfac/core/monitor/ExperimentIdentity.java | 36 + .../airavata/gfac/core/monitor/JobIdentity.java | 39 + .../airavata/gfac/core/monitor/MonitorID.java | 213 ++++++ .../gfac/core/monitor/TaskIdentity.java | 38 + .../gfac/core/monitor/WorkflowNodeIdentity.java | 37 + .../state/AbstractStateChangeRequest.java | 27 + .../state/ExperimentStatusChangeRequest.java | 63 ++ .../monitor/state/JobStatusChangeRequest.java | 80 ++ .../core/monitor/state/PublisherMessage.java | 26 + .../monitor/state/TaskStatusChangeRequest.java | 61 ++ .../state/WorkflowNodeStatusChangeRequest.java | 63 ++ .../gfac/core/notification/GFacNotifier.java | 42 ++ .../core/notification/MonitorPublisher.java | 47 ++ .../notification/events/ExecutionFailEvent.java | 35 + .../events/FinishExecutionEvent.java | 29 + .../events/FinishScheduleEvent.java | 29 + .../core/notification/events/GFacEvent.java | 39 + .../core/notification/events/JobIDEvent.java | 35 + .../events/StartExecutionEvent.java | 29 + .../notification/events/StartScheduleEvent.java | 29 + .../notification/events/StatusChangeEvent.java | 33 + .../notification/events/UnicoreJobIDEvent.java | 35 + .../notification/listeners/LoggingListener.java | 57 ++ .../listeners/WorkflowTrackingListener.java | 133 ++++ .../airavata/gfac/core/persistence/JobData.java | 55 ++ .../core/persistence/JobPersistenceManager.java | 76 ++ .../gfac/core/provider/AbstractProvider.java | 65 ++ .../gfac/core/provider/GFacProvider.java | 63 ++ .../gfac/core/provider/GFacProviderConfig.java | 51 ++ .../core/provider/GFacProviderException.java | 50 ++ .../gfac/core/provider/utils/DataIDType.java | 95 +++ .../gfac/core/provider/utils/ProviderUtils.java | 54 ++ .../provider/utils/ResourceRequirement.java | 34 + .../gfac/core/scheduler/HostScheduler.java | 35 + .../scheduler/impl/SimpleHostScheduler.java | 33 + .../airavata/gfac/core/utils/GFacUtils.java | 662 +++++++++++++++++ .../airavata/gfac/core/utils/OutputUtils.java | 150 ++++ .../java/org/apache/airavata/gfac/cpi/GFac.java | 49 -- .../org/apache/airavata/gfac/cpi/GFacImpl.java | 351 --------- .../airavata/gfac/handler/AbstractHandler.java | 38 - .../gfac/handler/AppDescriptorCheckHandler.java | 88 --- .../airavata/gfac/handler/GFacHandler.java | 34 - .../gfac/handler/GFacHandlerConfig.java | 51 -- .../gfac/handler/GFacHandlerException.java | 46 -- .../gfac/monitor/AbstractActivityListener.java | 27 - .../AiravataExperimentStatusUpdator.java | 81 -- .../gfac/monitor/AiravataJobStatusUpdator.java | 145 ---- .../gfac/monitor/AiravataTaskStatusUpdator.java | 113 --- .../AiravataWorkflowNodeStatusUpdator.java | 112 --- .../gfac/monitor/ExperimentIdentity.java | 36 - .../airavata/gfac/monitor/HostMonitorData.java | 69 -- .../airavata/gfac/monitor/JobIdentity.java | 39 - .../apache/airavata/gfac/monitor/MonitorID.java | 238 ------ .../airavata/gfac/monitor/MonitorManager.java | 347 --------- .../airavata/gfac/monitor/TaskIdentity.java | 38 - .../airavata/gfac/monitor/UserMonitorData.java | 76 -- .../gfac/monitor/WorkflowNodeIdentity.java | 37 - .../command/ExperimentCancelRequest.java | 38 - .../gfac/monitor/command/TaskCancelRequest.java | 52 -- .../monitor/core/AiravataAbstractMonitor.java | 46 -- .../gfac/monitor/core/MessageParser.java | 43 -- .../airavata/gfac/monitor/core/Monitor.java | 30 - .../airavata/gfac/monitor/core/PullMonitor.java | 64 -- .../airavata/gfac/monitor/core/PushMonitor.java | 60 -- .../gfac/monitor/event/MonitorPublisher.java | 47 -- .../exception/AiravataMonitorException.java | 37 - .../gfac/monitor/impl/LocalJobMonitor.java | 59 -- .../monitor/impl/pull/qstat/QstatMonitor.java | 262 ------- .../impl/pull/qstat/ResourceConnection.java | 151 ---- .../monitor/impl/push/amqp/AMQPMonitor.java | 263 ------- .../monitor/impl/push/amqp/BasicConsumer.java | 86 --- .../impl/push/amqp/JSONMessageParser.java | 78 -- .../impl/push/amqp/UnRegisterWorker.java | 68 -- .../state/AbstractStateChangeRequest.java | 27 - .../state/ExperimentStatusChangeRequest.java | 63 -- .../monitor/state/JobStatusChangeRequest.java | 80 -- .../gfac/monitor/state/JobStatusInfo.java | 48 -- .../gfac/monitor/state/PublisherMessage.java | 26 - .../monitor/state/TaskStatusChangeRequest.java | 61 -- .../state/WorkflowNodeStatusChangeRequest.java | 63 -- .../monitor/state/impl/AmazonJobStatusInfo.java | 39 - .../monitor/state/impl/GridJobStatusInfo.java | 40 - .../gfac/monitor/util/AMQPConnectionUtil.java | 77 -- .../airavata/gfac/monitor/util/CommonUtils.java | 172 ----- .../airavata/gfac/monitor/util/X509Helper.java | 161 ---- .../gfac/notification/GFacNotifier.java | 42 -- .../notification/events/ExecutionFailEvent.java | 35 - .../events/FinishExecutionEvent.java | 29 - .../events/FinishScheduleEvent.java | 29 - .../gfac/notification/events/GFacEvent.java | 39 - .../gfac/notification/events/JobIDEvent.java | 35 - .../events/StartExecutionEvent.java | 29 - .../notification/events/StartScheduleEvent.java | 29 - .../notification/events/StatusChangeEvent.java | 33 - .../notification/events/UnicoreJobIDEvent.java | 35 - .../listeners/GSISSHJobSubmissionListener.java | 55 -- .../notification/listeners/LoggingListener.java | 57 -- .../listeners/WorkflowTrackingListener.java | 133 ---- .../airavata/gfac/persistence/JobData.java | 55 -- .../gfac/persistence/JobPersistenceManager.java | 76 -- .../airavata/gfac/provider/GFacProvider.java | 68 -- .../gfac/provider/GFacProviderConfig.java | 51 -- .../gfac/provider/GFacProviderException.java | 51 -- .../gfac/provider/impl/AbstractProvider.java | 92 --- .../gfac/provider/utils/DataIDType.java | 95 --- .../gfac/provider/utils/ProviderUtils.java | 54 -- .../provider/utils/ResourceRequirement.java | 34 - .../airavata/gfac/scheduler/HostScheduler.java | 35 - .../scheduler/impl/SimpleHostScheduler.java | 33 - .../apache/airavata/gfac/utils/GFacUtils.java | 737 ------------------- .../apache/airavata/gfac/utils/OutputUtils.java | 150 ---- .../apache/airavata/job/AMQPMonitorTest.java | 175 ----- .../apache/airavata/job/GFacConfigXmlTest.java | 113 +++ .../job/QstatMonitorTestWithMyProxyAuth.java | 167 ----- .../apache/airavata/job/TestGlobalHandler.java | 32 + .../org/apache/airavata/job/TestInHandler.java | 32 + .../org/apache/airavata/job/TestOutHandler.java | 32 + .../org/apache/airavata/job/TestProvider.java | 47 ++ .../airavata/job/TestThreadedHandler.java | 39 + .../src/test/resources/gfac-config.xml | 101 +-- modules/gfac/gfac-ec2/pom.xml | 4 +- .../apache/airavata/gfac/ec2/EC2Provider.java | 20 +- .../airavata/gfac/ec2/EC2ProviderEvent.java | 2 +- .../airavata/gfac/ec2/util/AmazonEC2Util.java | 2 +- .../airavata/gfac/ec2/EC2ProviderTest.java | 8 +- .../gfac-ec2/src/test/resources/gfac-config.xml | 66 +- .../context/security/GSISecurityContext.java | 288 -------- .../apache/airavata/gfac/external/GridFtp.java | 558 -------------- .../airavata/gfac/gram/external/GridFtp.java | 558 ++++++++++++++ .../gram/handler/GramDirectorySetupHandler.java | 139 ++++ .../gfac/gram/handler/GridFTPInputHandler.java | 207 ++++++ .../gfac/gram/handler/GridFTPOutputHandler.java | 349 +++++++++ .../persistence/DBJobPersistenceManager.java | 225 ++++++ .../gfac/gram/provider/impl/GramProvider.java | 527 +++++++++++++ .../gfac/gram/security/GSISecurityContext.java | 275 +++++++ .../gram/util/GramJobSubmissionListener.java | 141 ++++ .../gfac/gram/util/GramProviderUtils.java | 113 +++ .../gfac/gram/util/GramRSLGenerator.java | 211 ++++++ .../gfac/gram/util/GridFTPContactInfo.java | 61 ++ .../gfac/handler/GramDirectorySetupHandler.java | 135 ---- .../gfac/handler/GridFTPInputHandler.java | 204 ----- .../gfac/handler/GridFTPOutputHandler.java | 347 --------- .../persistence/DBJobPersistenceManager.java | 223 ------ .../gfac/provider/impl/GramProvider.java | 526 ------------- .../gfac/util/GramJobSubmissionListener.java | 141 ---- .../airavata/gfac/util/GramProviderUtils.java | 114 --- .../airavata/gfac/util/GramRSLGenerator.java | 211 ------ .../airavata/gfac/util/GridFTPContactInfo.java | 61 -- .../impl/GFacBaseTestWithMyProxyAuth.java | 2 +- .../impl/GramProviderTestWithMyProxyAuth.java | 14 +- .../src/test/resources/gfac-config.xml | 15 +- modules/gfac/gfac-gsissh/pom.xml | 2 +- .../context/security/GSISecurityContext.java | 288 -------- .../handler/GSISSHDirectorySetupHandler.java | 103 +++ .../gfac/gsissh/handler/GSISSHInputHandler.java | 138 ++++ .../gsissh/handler/GSISSHOutputHandler.java | 239 ++++++ .../gsissh/provider/impl/GSISSHProvider.java | 138 ++++ .../gsissh/security/GSISecurityContext.java | 288 ++++++++ .../gfac/gsissh/util/GFACGSISSHUtils.java | 192 +++++ .../handler/GSISSHDirectorySetupHandler.java | 98 --- .../gfac/handler/GSISSHInputHandler.java | 137 ---- .../gfac/handler/GSISSHOutputHandler.java | 234 ------ .../gfac/provider/impl/GSISSHProvider.java | 114 --- .../airavata/gfac/util/GFACGSISSHUtils.java | 98 --- .../impl/GSISSHProviderTestWithMyProxyAuth.java | 10 +- .../GSISecurityContextTestWithMyProxyAuth.java | 2 +- .../src/test/resources/gfac-config.xml | 4 +- .../hadoop/handler/HDFSDataMovementHandler.java | 103 +++ .../hadoop/handler/HadoopDeploymentHandler.java | 276 +++++++ .../hadoop/provider/impl/HadoopProvider.java | 154 ++++ .../gfac/hadoop/provider/utils/HadoopUtils.java | 60 ++ .../gfac/handler/HDFSDataMovementHandler.java | 102 --- .../gfac/handler/HadoopDeploymentHandler.java | 274 ------- .../gfac/provider/impl/HadoopProvider.java | 153 ---- .../gfac/provider/utils/HadoopUtils.java | 60 -- .../src/test/resources/gfac-config.xml | 68 +- .../handler/LocalDirectorySetupHandler.java | 62 -- .../handler/LocalDirectorySetupHandler.java | 63 ++ .../gfac/local/provider/impl/LocalProvider.java | 249 +++++++ .../local/utils/InputStreamToFileWriter.java | 68 ++ .../airavata/gfac/local/utils/InputUtils.java | 39 + .../gfac/local/utils/LocalProviderUtil.java | 54 ++ .../gfac/provider/impl/LocalProvider.java | 240 ------ .../gfac/utils/InputStreamToFileWriter.java | 68 -- .../apache/airavata/gfac/utils/InputUtils.java | 39 - .../airavata/gfac/utils/LocalProviderUtil.java | 54 -- .../gfac/services/impl/LocalProviderTest.java | 57 +- .../src/test/resources/gfac-config.xml | 69 +- modules/gfac/gfac-monitor/pom.xml | 186 +++++ .../airavata/gfac/monitor/HPCMonitorID.java | 248 +++++++ .../airavata/gfac/monitor/HostMonitorData.java | 70 ++ .../airavata/gfac/monitor/UserMonitorData.java | 76 ++ .../command/ExperimentCancelRequest.java | 38 + .../gfac/monitor/command/TaskCancelRequest.java | 52 ++ .../monitor/core/AiravataAbstractMonitor.java | 46 ++ .../gfac/monitor/core/MessageParser.java | 43 ++ .../airavata/gfac/monitor/core/Monitor.java | 30 + .../airavata/gfac/monitor/core/PullMonitor.java | 64 ++ .../airavata/gfac/monitor/core/PushMonitor.java | 60 ++ .../exception/AiravataMonitorException.java | 37 + .../handlers/GridPullMonitorHandler.java | 101 +++ .../monitor/impl/pull/qstat/HPCPullMonitor.java | 291 ++++++++ .../impl/pull/qstat/ResourceConnection.java | 115 +++ .../monitor/impl/push/amqp/AMQPMonitor.java | 263 +++++++ .../monitor/impl/push/amqp/BasicConsumer.java | 86 +++ .../impl/push/amqp/JSONMessageParser.java | 78 ++ .../impl/push/amqp/UnRegisterWorker.java | 68 ++ .../gfac/monitor/util/AMQPConnectionUtil.java | 77 ++ .../airavata/gfac/monitor/util/CommonUtils.java | 205 ++++++ .../airavata/gfac/monitor/util/X509Helper.java | 161 ++++ .../src/main/resources/errors.properties | 197 +++++ .../src/main/resources/schema/AccessPolicy.json | 13 + .../src/main/resources/schema/Activity.json | 31 + .../src/main/resources/schema/AdminDomain.json | 51 ++ .../schema/ApplicationEnvironment.json | 86 +++ .../resources/schema/ApplicationHandle.json | 21 + .../src/main/resources/schema/Benchmark.json | 21 + .../resources/schema/ComputingActivity.json | 165 +++++ .../resources/schema/ComputingEndpoint.json | 44 ++ .../main/resources/schema/ComputingManager.json | 117 +++ .../main/resources/schema/ComputingService.json | 32 + .../main/resources/schema/ComputingShare.json | 182 +++++ .../src/main/resources/schema/Contact.json | 32 + .../src/main/resources/schema/DataStore.json | 30 + .../src/main/resources/schema/Domain.json | 30 + .../src/main/resources/schema/Endpoint.json | 147 ++++ .../src/main/resources/schema/Entity.json | 35 + .../resources/schema/ExecutionEnvironment.json | 115 +++ .../src/main/resources/schema/Glue2.json | 246 +++++++ .../src/main/resources/schema/Location.json | 47 ++ .../src/main/resources/schema/Manager.json | 28 + .../main/resources/schema/MappingPolicy.json | 13 + .../src/main/resources/schema/Policy.json | 27 + .../src/main/resources/schema/Resource.json | 27 + .../src/main/resources/schema/Service.json | 75 ++ .../src/main/resources/schema/Share.json | 45 ++ .../resources/schema/StorageAccessProtocol.json | 32 + .../main/resources/schema/StorageEndpoint.json | 8 + .../main/resources/schema/StorageManager.json | 8 + .../main/resources/schema/StorageService.json | 22 + .../schema/StorageServiceCapacity.json | 33 + .../src/main/resources/schema/StorageShare.json | 65 ++ .../resources/schema/StorageShareCapacity.json | 33 + .../resources/schema/ToComputingService.json | 32 + .../main/resources/schema/ToStorageService.json | 25 + .../src/main/resources/schema/UserDomain.json | 58 ++ .../src/main/resources/service.properties | 58 ++ .../apache/airavata/job/AMQPMonitorTest.java | 178 +++++ .../job/QstatMonitorTestWithMyProxyAuth.java | 171 +++++ .../src/test/resources/PBSTemplate.xslt | 73 ++ .../gfac-monitor/src/test/resources/echo.bat | 22 + .../src/test/resources/gfac-config.xml | 65 ++ .../src/test/resources/logging.properties | 42 ++ .../context/security/SSHSecurityContext.java | 120 --- .../gfac/handler/AdvancedSCPInputHandler.java | 150 ---- .../gfac/handler/AdvancedSCPOutputHandler.java | 126 ---- .../gfac/handler/SSHDirectorySetupHandler.java | 102 --- .../airavata/gfac/handler/SSHInputHandler.java | 140 ---- .../airavata/gfac/handler/SSHOutputHandler.java | 218 ------ .../gfac/provider/impl/SSHProvider.java | 309 -------- .../ssh/handler/AdvancedSCPInputHandler.java | 152 ++++ .../ssh/handler/AdvancedSCPOutputHandler.java | 130 ++++ .../ssh/handler/SSHDirectorySetupHandler.java | 107 +++ .../gfac/ssh/handler/SSHInputHandler.java | 143 ++++ .../gfac/ssh/handler/SSHOutputHandler.java | 226 ++++++ .../gfac/ssh/provider/impl/SSHProvider.java | 310 ++++++++ .../gfac/ssh/security/SSHSecurityContext.java | 120 +++ .../airavata/gfac/ssh/util/GFACSSHUtils.java | 179 +++++ .../apache/airavata/gfac/util/GFACSSHUtils.java | 88 --- .../services/impl/BigRed2TestWithSSHAuth.java | 11 +- .../impl/SSHProviderTestWithSSHAuth.java | 10 +- .../gfac-ssh/src/test/resources/gfac-config.xml | 38 +- modules/gfac/pom.xml | 1 + .../src/test/resources/gfac-config.xml | 84 ++- .../server/OrchestratorServerHandler.java | 17 - .../src/main/resources/gfac-config.xml | 84 ++- .../src/test/resources/gfac-config.xml | 4 +- .../core/context/OrchestratorContext.java | 11 - .../core/impl/EmbeddedGFACJobSubmitter.java | 17 +- .../orchestrator/core/job/JobSubmitter.java | 11 +- .../cpi/impl/SimpleOrchestratorImpl.java | 171 +---- .../orchestrator/core/NewOrchestratorTest.java | 2 + .../src/test/resources/gfac-config.xml | 84 ++- 360 files changed, 21022 insertions(+), 18335 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/ee79b70b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java ---------------------------------------------------------------------- diff --cc modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java index 0000000,cff1367..6acddf8 mode 000000,100644..100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java @@@ -1,0 -1,105 +1,150 @@@ + /* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + package org.apache.airavata.gfac.core.utils; + + import java.util.HashMap; ++import java.util.List; + import java.util.Map; + import java.util.Set; + import java.util.regex.Matcher; + import java.util.regex.Pattern; + + import org.apache.airavata.common.utils.StringUtil; + import org.apache.airavata.commons.gfac.type.ActualParameter; + import org.apache.airavata.commons.gfac.type.MappingFactory; ++import org.apache.airavata.model.workspace.experiment.DataObjectType; ++import org.apache.airavata.schemas.gfac.DataType; + import org.apache.airavata.gfac.core.handler.GFacHandlerException; + import org.apache.airavata.schemas.gfac.StdErrParameterType; + import org.apache.airavata.schemas.gfac.StdOutParameterType; + + public class OutputUtils { + private static String regexPattern = "\\s*=\\s*(.*)\\r?\\n"; + + public static Map fillOutputFromStdout(Map output, String stdout, String stderr) throws Exception { + + if (stdout == null || stdout.equals("")){ + throw new GFacHandlerException("Standard output is empty."); + } + + Map result = new HashMap(); + Set keys = output.keySet(); + for (String paramName : keys) { + ActualParameter actual = (ActualParameter) output.get(paramName); + // if parameter value is not already set, we let it go + + if (actual == null) { + continue; + } + if ("StdOut".equals(actual.getType().getType().toString())) { + ((StdOutParameterType) actual.getType()).setValue(stdout); + result.put(paramName, actual); + } else if ("StdErr".equals(actual.getType().getType().toString())) { + ((StdErrParameterType) actual.getType()).setValue(stderr); + result.put(paramName, actual); + }else if("URI".equals(actual.getType().getType().toString())){ + continue; + } + else { + String parseStdout = parseStdout(stdout, paramName); + if (parseStdout != null) { + MappingFactory.fromString(actual, parseStdout); + result.put(paramName, actual); + } + } + } + + return result; + } ++ ++ public static void fillOutputFromStdout1(Map output, String stdout, String stderr,List outputArray) throws Exception { ++ ++ if (stdout == null || stdout.equals("")){ ++ throw new GFacHandlerException("Standard output is empty."); ++ } ++ ++ Set keys = output.keySet(); ++ for (String paramName : keys) { ++ ActualParameter actual = (ActualParameter) output.get(paramName); ++ // if parameter value is not already set, we let it go ++ ++ if (actual == null) { ++ continue; ++ } ++ if ("StdOut".equals(actual.getType().getType().toString())) { ++ DataObjectType out = new DataObjectType(); ++ out.setKey(paramName); ++ out.setType(DataType.STD_OUT.toString()); ++ out.setValue(stdout); ++ outputArray.add(out); ++ } else if ("StdErr".equals(actual.getType().getType().toString())) { ++ DataObjectType out = new DataObjectType(); ++ out.setKey(paramName); ++ out.setType(DataType.STD_ERR.toString()); ++ out.setValue(stderr); ++ outputArray.add(out); ++ }else if("URI".equals(actual.getType().getType().toString())){ ++ continue; ++ } ++ else { ++ String parseStdout = parseStdout(stdout, paramName); ++ if (parseStdout != null) { ++ DataObjectType out = new DataObjectType(); ++ out.setKey(paramName); ++ out.setType(DataType.STRING.toString()); ++ out.setValue(parseStdout); ++ outputArray.add(out); ++ } ++ } ++ } ++ } + + private static String parseStdout(String stdout, String outParam) throws Exception { + String regex = Pattern.quote(outParam) + regexPattern; + String match = null; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(stdout); + while (matcher.find()) { + match = matcher.group(1); + } + if (match != null) { + match = match.trim(); + return match; + } else { + throw new Exception("Data for the output parameter '" + outParam + "' was not found"); + } + } + + public static String[] parseStdoutArray(String stdout, String outParam) throws Exception { + String regex = Pattern.quote(outParam) + regexPattern; + StringBuffer match = new StringBuffer(); + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(stdout); + while (matcher.find()) { + match.append(matcher.group(1) + StringUtil.DELIMETER); + } + if (match != null) { + return StringUtil.getElementsFromString(match.toString()); + } else { + throw new Exception("Data for the output parameter '" + outParam + "' was not found"); + } + } + } http://git-wip-us.apache.org/repos/asf/airavata/blob/ee79b70b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java ---------------------------------------------------------------------- diff --cc modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java index 0000000,aefa80f..8c53d87 mode 000000,100644..100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java @@@ -1,0 -1,221 +1,239 @@@ + /* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + package org.apache.airavata.gfac.gsissh.handler; + ++import java.io.File; ++import java.io.IOException; ++import java.lang.reflect.Type; ++import java.util.ArrayList; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; ++import java.util.Set; ++ + import net.schmizz.sshj.connection.ConnectionException; + import net.schmizz.sshj.transport.TransportException; ++ + import org.apache.airavata.common.exception.ApplicationSettingsException; + import org.apache.airavata.common.utils.Constants; + import org.apache.airavata.commons.gfac.type.ActualParameter; + import org.apache.airavata.commons.gfac.type.ApplicationDescription; + import org.apache.airavata.gfac.GFacException; + import org.apache.airavata.gfac.core.context.JobExecutionContext; + import org.apache.airavata.gfac.core.handler.AbstractHandler; + import org.apache.airavata.gfac.core.handler.GFacHandlerException; + import org.apache.airavata.gfac.core.provider.GFacProviderException; + import org.apache.airavata.gfac.core.utils.GFacUtils; + import org.apache.airavata.gfac.core.utils.OutputUtils; + import org.apache.airavata.gfac.gsissh.security.GSISecurityContext; + import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils; + import org.apache.airavata.gsi.ssh.api.Cluster; + import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; -import org.apache.airavata.model.workspace.experiment.*; ++import org.apache.airavata.model.workspace.experiment.CorrectiveAction; ++import org.apache.airavata.model.workspace.experiment.DataObjectType; ++import org.apache.airavata.model.workspace.experiment.DataTransferDetails; ++import org.apache.airavata.model.workspace.experiment.ErrorCategory; ++import org.apache.airavata.model.workspace.experiment.JobDetails; ++import org.apache.airavata.model.workspace.experiment.TaskDetails; ++import org.apache.airavata.model.workspace.experiment.TransferState; ++import org.apache.airavata.model.workspace.experiment.TransferStatus; + import org.apache.airavata.registry.cpi.ChildDataType; + import org.apache.airavata.registry.cpi.DataType; + import org.apache.airavata.registry.cpi.RegistryException; + import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; + import org.apache.airavata.schemas.gfac.GsisshHostType; + import org.apache.airavata.schemas.gfac.URIParameterType; + import org.apache.xmlbeans.XmlException; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; ++import com.google.gson.GsonBuilder; ++import com.google.gson.reflect.TypeToken; + + public class GSISSHOutputHandler extends AbstractHandler { + private static final Logger log = LoggerFactory.getLogger(GSISSHOutputHandler.class); + + public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException { + if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context + // so attempting to get it from the registry + if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) { + log.warn("During the out handler chain jobExecution context came null, so trying to handler"); + ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); + TaskDetails taskData = null; + try { + taskData = (TaskDetails) jobExecutionContext.getRegistry().get(DataType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID()); + } catch (RegistryException e) { + log.error("Error retrieving job details from Registry"); + throw new GFacHandlerException("Error retrieving job details from Registry", e); + } + JobDetails jobDetails = taskData.getJobDetailsList().get(0); + String jobDescription = jobDetails.getJobDescription(); + if (jobDescription != null) { + JobDescriptor jobDescriptor = null; + try { + jobDescriptor = JobDescriptor.fromXML(jobDescription); + } catch (XmlException e1) { + e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + applicationDeploymentDescription.getType().setScratchWorkingDirectory( + jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory()); + applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory()); + applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory()); + applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile()); + applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile()); + } + } + } + try { + if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) { + + GFACGSISSHUtils.addSecurityContext(jobExecutionContext); + } + } catch (ApplicationSettingsException e) { + log.error(e.getMessage()); + throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage()); + } catch (GFacException e) { + log.error(e.getMessage()); + throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage()); + } + super.invoke(jobExecutionContext); + DataTransferDetails detail = new DataTransferDetails(); + TransferStatus status = new TransferStatus(); + + ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext() + .getApplicationDeploymentDescription().getType(); + try { + Cluster cluster = null; + if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) { + cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster(); + } else { + cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster(); + } + if (cluster == null) { + throw new GFacProviderException("Security context is not set properly"); + } else { + log.info("Successfully retrieved the Security Context"); + } + + // Get the Stdouts and StdErrs + String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName()); + + TaskDetails taskData = jobExecutionContext.getTaskData(); + String outputDataDir = null; + File localStdOutFile; + File localStdErrFile; + + if (taskData.getAdvancedOutputDataHandling() != null) { + outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir(); + } + if (outputDataDir == null) { + outputDataDir = File.separator + "tmp"; + } + outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID(); + (new File(outputDataDir)).mkdirs(); + + + localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout"); + localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr"); + // cluster.makeDirectory(outputDataDir); + cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath()); + Thread.sleep(1000); + cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath()); + Thread.sleep(1000); + + String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath()); + String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath()); + status.setTransferState(TransferState.COMPLETE); + detail.setTransferStatus(status); + detail.setTransferDescription("STDOUT:" + stdOutStr); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + + status.setTransferState(TransferState.COMPLETE); + detail.setTransferStatus(status); + detail.setTransferDescription("STDERR:" + stdErrStr); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + + - Map stringMap = new HashMap(); ++ List outputArray = new ArrayList(); + Map output = jobExecutionContext.getOutMessageContext().getParameters(); + Set keys = output.keySet(); + for (String paramName : keys) { + ActualParameter actualParameter = (ActualParameter) output.get(paramName); + if ("URI".equals(actualParameter.getType().getType().toString())) { + + List outputList = cluster.listDirectory(app.getOutputDataDirectory()); + if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); ++ OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr, outputArray); ++ break; + } else { + String valueList = outputList.get(0); + cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir); + jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList); - ((URIParameterType) actualParameter.getType()).setValue(valueList); - stringMap = new HashMap(); - stringMap.put(paramName, actualParameter); ++ DataObjectType dataObjectType = new DataObjectType(); ++ dataObjectType.setValue(valueList); ++ dataObjectType.setKey(paramName); ++ dataObjectType.setType(org.apache.airavata.schemas.gfac.DataType.URI.toString()); ++ outputArray.add(dataObjectType); + } + } else { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); ++ OutputUtils.fillOutputFromStdout1(output, stdOutStr,stdErrStr, outputArray); ++ break; + } + } - if (stringMap == null || stringMap.isEmpty()) { ++ if (outputArray == null || outputArray.isEmpty()) { + throw new GFacHandlerException( + "Empty Output returned from the Application, Double check the application" + + "and ApplicationDescriptor output Parameter Names"); + } - status.setTransferState(TransferState.DOWNLOAD); - detail.setTransferStatus(status); - registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); - + app.setStandardError(localStdErrFile.getAbsolutePath()); + app.setStandardOutput(localStdOutFile.getAbsolutePath()); + app.setOutputDataDirectory(outputDataDir); ++ status.setTransferState(TransferState.DOWNLOAD); ++ detail.setTransferStatus(status); ++ detail.setTransferDescription(outputDataDir); ++ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); ++ registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); + } catch (XmlException e) { + throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e); + } catch (ConnectionException e) { + throw new GFacHandlerException(e.getMessage(), e); + } catch (TransportException e) { + throw new GFacHandlerException(e.getMessage(), e); + } catch (IOException e) { + throw new GFacHandlerException(e.getMessage(), e); + } catch (Exception e) { + try { + status.setTransferState(TransferState.FAILED); + detail.setTransferStatus(status); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE); + } catch (Exception e1) { + throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage()); + } + throw new GFacHandlerException("Error in retrieving results", e); + } + + } + + public void initProperties(Map properties) throws GFacHandlerException { + + } + } http://git-wip-us.apache.org/repos/asf/airavata/blob/ee79b70b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java ---------------------------------------------------------------------- diff --cc modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java index 0000000,902c09f..66296f2 mode 000000,100644..100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java @@@ -1,0 -1,220 +1,226 @@@ + /* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + package org.apache.airavata.gfac.ssh.handler; + + import java.io.File; + import java.io.IOException; ++import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Set; + + import net.schmizz.sshj.connection.ConnectionException; + import net.schmizz.sshj.transport.TransportException; + + import org.apache.airavata.common.exception.ApplicationSettingsException; + import org.apache.airavata.common.utils.Constants; + import org.apache.airavata.commons.gfac.type.ActualParameter; + import org.apache.airavata.commons.gfac.type.ApplicationDescription; + import org.apache.airavata.gfac.GFacException; + import org.apache.airavata.gfac.core.context.JobExecutionContext; + import org.apache.airavata.gfac.core.handler.AbstractHandler; + import org.apache.airavata.gfac.core.handler.GFacHandlerException; + import org.apache.airavata.gfac.core.provider.GFacProviderException; + import org.apache.airavata.gfac.core.utils.GFacUtils; + import org.apache.airavata.gfac.core.utils.OutputUtils; + import org.apache.airavata.gfac.ssh.security.SSHSecurityContext; + import org.apache.airavata.gfac.ssh.util.GFACSSHUtils; + import org.apache.airavata.gsi.ssh.api.Cluster; + import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; + import org.apache.airavata.model.workspace.experiment.*; + import org.apache.airavata.registry.cpi.ChildDataType; + import org.apache.airavata.registry.cpi.DataType; + import org.apache.airavata.registry.cpi.RegistryException; + import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; + import org.apache.airavata.schemas.gfac.GsisshHostType; + import org.apache.airavata.schemas.gfac.URIParameterType; + import org.apache.xmlbeans.XmlException; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + public class SSHOutputHandler extends AbstractHandler { + private static final Logger log = LoggerFactory.getLogger(SSHOutputHandler.class); + + public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException { + if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context + // so attempting to get it from the registry + if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) { // this is because we don't have the right jobexecution context + // so attempting to get it from the registry + log.warn("During the out handler chain jobExecution context came null, so trying to handler"); + ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); + TaskDetails taskData = null; + try { + taskData = (TaskDetails) registry.get(DataType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID()); + } catch (RegistryException e) { + log.error("Error retrieving job details from Registry"); + throw new GFacHandlerException("Error retrieving job details from Registry", e); + } + JobDetails jobDetails = taskData.getJobDetailsList().get(0); + String jobDescription = jobDetails.getJobDescription(); + if (jobDescription != null) { + JobDescriptor jobDescriptor = null; + try { + jobDescriptor = JobDescriptor.fromXML(jobDescription); + } catch (XmlException e1) { + e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + applicationDeploymentDescription.getType().setScratchWorkingDirectory( + jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory()); + applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory()); + applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory()); + applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile()); + applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile()); + } + } + } + + try { + if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) { + + GFACSSHUtils.addSecurityContext(jobExecutionContext); + } + } catch (ApplicationSettingsException e) { + log.error(e.getMessage()); + throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage()); + } catch (GFacException e1) { + log.error(e1.getMessage()); + throw new GFacHandlerException("Error while creating SSHSecurityContext", e1, e1.getLocalizedMessage()); + } + + super.invoke(jobExecutionContext); + DataTransferDetails detail = new DataTransferDetails(); + TransferStatus status = new TransferStatus(); + + ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext() + .getApplicationDeploymentDescription().getType(); + try { + Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster(); + if (cluster == null) { + throw new GFacProviderException("Security context is not set properly"); + } else { + log.info("Successfully retrieved the Security Context"); + } + + // Get the Stdouts and StdErrs + String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName()); + + TaskDetails taskData = jobExecutionContext.getTaskData(); + String outputDataDir = null; + File localStdOutFile; + File localStdErrFile; + + if (taskData.getAdvancedOutputDataHandling() != null) { + outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir(); + } + if (outputDataDir == null) { + outputDataDir = File.separator + "tmp"; + } + outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID(); + (new File(outputDataDir)).mkdirs(); + + + localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout"); + localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr"); + // cluster.makeDirectory(outputDataDir); + cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath()); + Thread.sleep(1000); + cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath()); + Thread.sleep(1000); + + String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath()); + String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath()); + status.setTransferState(TransferState.COMPLETE); + detail.setTransferStatus(status); + detail.setTransferDescription("STDOUT:" + stdOutStr); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + + status.setTransferState(TransferState.COMPLETE); + detail.setTransferStatus(status); + detail.setTransferDescription("STDERR:" + stdErrStr); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + + - Map stringMap = new HashMap(); ++ List outputArray = new ArrayList(); + Map output = jobExecutionContext.getOutMessageContext().getParameters(); + Set keys = output.keySet(); + for (String paramName : keys) { + ActualParameter actualParameter = (ActualParameter) output.get(paramName); + if ("URI".equals(actualParameter.getType().getType().toString())) { + + List outputList = cluster.listDirectory(app.getOutputDataDirectory()); + if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); ++ OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray); ++ break; + } else { + String valueList = outputList.get(0); + cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir); + jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList); - ((URIParameterType) actualParameter.getType()).setValue(valueList); - stringMap = new HashMap(); - stringMap.put(paramName, actualParameter); ++ DataObjectType dataObjectType = new DataObjectType(); ++ dataObjectType.setValue(valueList); ++ dataObjectType.setKey(paramName); ++ dataObjectType.setType(org.apache.airavata.schemas.gfac.DataType.URI.toString()); ++ outputArray.add(dataObjectType); + } + } else { - stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); ++ OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray); + } + } - if (stringMap == null || stringMap.isEmpty()) { ++ if (outputArray == null || outputArray.isEmpty()) { + throw new GFacHandlerException( + "Empty Output returned from the Application, Double check the application" + + "and ApplicationDescriptor output Parameter Names"); + } - status.setTransferState(TransferState.DOWNLOAD); - detail.setTransferStatus(status); - registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); - + app.setStandardError(localStdErrFile.getAbsolutePath()); + app.setStandardOutput(localStdOutFile.getAbsolutePath()); + app.setOutputDataDirectory(outputDataDir); ++ status.setTransferState(TransferState.DOWNLOAD); ++ detail.setTransferStatus(status); ++ detail.setTransferDescription(outputDataDir); ++ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); ++ registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); ++ + } catch (XmlException e) { + throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e); + } catch (ConnectionException e) { + throw new GFacHandlerException(e.getMessage(), e); + } catch (TransportException e) { + throw new GFacHandlerException(e.getMessage(), e); + } catch (IOException e) { + throw new GFacHandlerException(e.getMessage(), e); + } catch (Exception e) { + try { + status.setTransferState(TransferState.FAILED); + detail.setTransferStatus(status); + registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); + GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE); + } catch (Exception e1) { + throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage()); + } + throw new GFacHandlerException("Error in retrieving results", e); + } + + } + + public void initProperties(Map properties) throws GFacHandlerException { + + } + }