airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lah...@apache.org
Subject [1/2] git commit: more improvements to experiment cancel feature
Date Tue, 19 Aug 2014 09:42:13 GMT
Repository: airavata
Updated Branches:
  refs/heads/master d0fcdb172 -> 414f2b9cf


more improvements to experiment cancel feature


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/eb9bda87
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/eb9bda87
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/eb9bda87

Branch: refs/heads/master
Commit: eb9bda8736fce7c0ca4f367248d5c896ed9f0f34
Parents: 011f019
Author: lahiru <lahiru@apache.org>
Authored: Tue Aug 19 15:11:36 2014 +0530
Committer: lahiru <lahiru@apache.org>
Committed: Tue Aug 19 15:11:36 2014 +0530

----------------------------------------------------------------------
 .../client/samples/CancelExperiments.java       |  2 +-
 .../client/samples/CreateLaunchExperiment.java  |  4 +-
 .../airavata/gfac/core/cpi/BetterGfacImpl.java  |  6 ---
 .../core/monitor/AiravataTaskStatusUpdator.java |  2 +
 .../airavata/gfac/core/utils/GFacUtils.java     | 48 ++++++++++---------
 .../gsissh/provider/impl/GSISSHProvider.java    | 19 +++++---
 .../gfac/ssh/provider/impl/SSHProvider.java     | 49 ++++++++++++++++++--
 .../server/OrchestratorServerHandler.java       |  3 ++
 .../util/OrchestratorRecoveryHandler.java       |  7 ++-
 .../core/impl/GFACServiceJobSubmitter.java      |  4 +-
 .../gsi/ssh/impl/GSISSHAbstractCluster.java     |  2 +-
 11 files changed, 102 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CancelExperiments.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CancelExperiments.java
b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CancelExperiments.java
index b1b1503..d3b7e46 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CancelExperiments.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CancelExperiments.java
@@ -48,7 +48,7 @@ public class CancelExperiments {
             AiravataUtils.setExecutionAsClient();
             client = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
 
-            String expeId = "echoExperiment_06a24a33-6c39-4349-9a47-f16633a2152b";
+            String expeId = "echoExperiment_31c132fd-87ea-4781-803c-ae5f04a79baf";
             terminateExperiment(client,expeId);
 
 //            System.out.println("retrieved exp id : " + experiment.getExperimentID());

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 9c82eac..9d54583 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -64,7 +64,7 @@ public class CreateLaunchExperiment {
     private static Airavata.Client client;
     private static String localHostAppId = "localhost_3b5962d3-5e7e-4a97-9d1f-25c5ec436ba5,SimpleEcho0_44c34394-ca27-4fa9-bb2d-87f95a02352a";
     private static String sshHostAppId;
-    private static String pbsEchoAppId = "trestles.sdsc.edu_6dc7de3d-0d18-4933-bd96-b40c425f44c5,SimpleEcho2_8cc763c9-c57c-4a23-890c-4d3cee413c68";
+    private static String pbsEchoAppId = "trestles.sdsc.edu_06f378d4-e25e-40f9-bd0e-c252e46177be,SimpleEcho2_2c836f69-3455-4b57-a083-78e2b19e1e43";
     private static String pbsWRFAppId = "trestles.sdsc.edu_66caf560-497d-4316-b1e9-adadc5817b20,WRF_61490c54-dacf-4bc4-874c-ad5909a65afb";
     private static String slurmAppId = "stampede.tacc.xsede.org_b2ef59cb-f626-4767-9ca0-601f94c42ba4,SimpleEcho3_b81c2559-a088-42a3-84ce-40119d874918";
     private static String sgeAppId;
@@ -82,7 +82,7 @@ public class CreateLaunchExperiment {
             client = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
             System.out.println("API version is " + client.getAPIVersion());
 //            getExperiment(client, "WRFExperiment_2a2de26c-7f74-47c9-8e14-40e50dedfe0f");
-            addDescriptors();
+//            addDescriptors();
 
 ////            final String expId = createExperimentForSSHHost(airavata);
             final String expId = createExperimentForTrestles(client);

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index 624f9ab..9415625 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -565,15 +565,11 @@ public class BetterGfacImpl implements GFac,Watcher {
                 // In this scenario We do everything from the beginning
                 log.info("Job is not yet submitted, so nothing much to do except changing
the registry entry " +
                         " and stop the execution chain");
-                //todo update registry and find a way to stop the execution chain
-                GFacUtils.setExperimentCancel(jobExecutionContext.getExperimentID(), jobExecutionContext.getTaskData().getTaskID(),
zk);
             } else if (stateVal >= 8) {
                 log.error("This experiment is almost finished, so cannot cancel this experiment");
                 ZKUtil.deleteRecursive(zk,
                         AiravataZKUtils.getExpZnodePath(jobExecutionContext.getExperimentID(),
jobExecutionContext.getTaskData().getTaskID()));
             } else {
-                // Now we know this is an old Job, so we have to handle things gracefully
-                GFacUtils.setExperimentCancel(jobExecutionContext.getExperimentID(), jobExecutionContext.getTaskData().getTaskID(),
zk);
                 log.info("Job is in a position to perform a proper cancellation");
                 try {
                     Scheduler.schedule(jobExecutionContext);
@@ -812,11 +808,9 @@ public class BetterGfacImpl implements GFac,Watcher {
         GFacProvider provider = jobExecutionContext.getProvider();
         if (provider != null) {
             monitorPublisher.publish(new GfacExperimentStateChangeRequest(new MonitorID(jobExecutionContext),
GfacExperimentState.PROVIDERINVOKING));
-            GFacUtils.createPluginZnode(zk, jobExecutionContext, provider.getClass().getName());
             initProvider(provider, jobExecutionContext);
             cancelProvider(provider, jobExecutionContext);
             disposeProvider(provider, jobExecutionContext);
-            GFacUtils.updatePluginState(zk, jobExecutionContext, provider.getClass().getName(),
GfacPluginState.COMPLETED);
             monitorPublisher.publish(new GfacExperimentStateChangeRequest(new MonitorID(jobExecutionContext),
GfacExperimentState.PROVIDERINVOKED));
         }
         if (GFacUtils.isSynchronousMode(jobExecutionContext)) {

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/AiravataTaskStatusUpdator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/AiravataTaskStatusUpdator.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/AiravataTaskStatusUpdator.java
index 6457435..9f00e11 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/AiravataTaskStatusUpdator.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/AiravataTaskStatusUpdator.java
@@ -106,6 +106,8 @@ public class AiravataTaskStatusUpdator implements AbstractActivityListener
{
         if(!TaskState.CANCELED.equals(details.getTaskStatus().getExecutionState())
                 && !TaskState.CANCELING.equals(details.getTaskStatus().getExecutionState())){
             status.setExecutionState(state);
+        }else{
+            status.setExecutionState(details.getTaskStatus().getExecutionState());
         }
         status.setTimeOfStateChange(Calendar.getInstance().getTimeInMillis());
         details.setTaskStatus(status);

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
index ed4dd14..413bc13 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/GFacUtils.java
@@ -1012,9 +1012,10 @@ public class GFacUtils {
 		String experimentPath = experimentNode + File.separator + pickedChild;
 		String newExpNode = experimentPath + File.separator + experimentID
 				+ "+" + taskID;
-		Stat exists1 = zk.exists(newExpNode, false);
-		String foundExperimentPath = null;
-		if (exists1 == null) {
+        Stat exists1 = zk.exists(newExpNode, false);
+        String experimentEntry = GFacUtils.findExperimentEntry(experimentID, taskID, zk);
+        String foundExperimentPath = null;
+		if (exists1 == null && experimentEntry == null) {  // this means this is a very
new experiment
 			List<String> runningGfacNodeNames = AiravataZKUtils
 					.getAllGfacNodeNames(zk); // here we take old gfac servers
 												// too
@@ -1089,24 +1090,27 @@ public class GFacUtils {
 				log.info("Deleting experiment data: " + foundExperimentPath);
 				ZKUtil.deleteRecursive(zk, foundExperimentPath);
 			}
-		} else {
-			log.error("ExperimentID: " + experimentID + " taskID: " + taskID
-					+ " is already running by this Gfac instance");
-			List<String> runningGfacNodeNames = AiravataZKUtils
-					.getAllGfacNodeNames(zk); // here we take old gfac servers
-												// too
-			for (String gfacServerNode : runningGfacNodeNames) {
-				if (!gfacServerNode.equals(pickedChild)) {
-					foundExperimentPath = experimentNode + File.separator
-							+ gfacServerNode + File.separator + experimentID
-							+ "+" + taskID;
-					break;
-				}
-			}
-			ZKUtil.deleteRecursive(zk, foundExperimentPath);
-			return false;
-		}
-		return true;
+		}else if(experimentEntry != null && GFacUtils.isCancelled(experimentID,taskID,zk)
){
+            // this happens when a cancel request comes to a differnt gfac node, in this
case we do not move gfac experiment
+            // node to gfac node specific location, because original request execution will
fail with errors
+            return true;
+        } else {
+            log.error("ExperimentID: " + experimentID + " taskID: " + taskID
+                    + " is already running by this Gfac instance");
+            List<String> runningGfacNodeNames = AiravataZKUtils
+                    .getAllGfacNodeNames(zk); // here we take old gfac servers
+            // too
+            for (String gfacServerNode : runningGfacNodeNames) {
+                if (!gfacServerNode.equals(pickedChild)) {
+                    foundExperimentPath = experimentNode + File.separator
+                            + gfacServerNode + File.separator + experimentID
+                            + "+" + taskID;
+                    break;
+                }
+            }
+            ZKUtil.deleteRecursive(zk, foundExperimentPath);
+        }
+        return true;
 	}
 
     public static String findExperimentEntry(String experimentID,
@@ -1152,7 +1156,7 @@ public class GFacUtils {
         }else {
             Stat exists = zk.exists(experimentEntry, false);
             if (exists != null) {
-                String operation = new String(zk.getData(experimentEntry, false, exists));
+                String operation = new String(zk.getData(experimentEntry+File.separator+"operation",
false, exists));
                 if ("cancel".equals(operation)) {
                     return true;
                 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/provider/impl/GSISSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/provider/impl/GSISSHProvider.java
b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/provider/impl/GSISSHProvider.java
index 74975df..d6981f3 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/provider/impl/GSISSHProvider.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/provider/impl/GSISSHProvider.java
@@ -61,6 +61,16 @@ public class GSISSHProvider extends AbstractRecoverableProvider {
 
     public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,
GFacException {
         super.initialize(jobExecutionContext);
+        try {
+            if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)
== null) {
+                GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
+            }
+        } catch (ApplicationSettingsException e) {
+            log.error(e.getMessage());
+            throw new GFacHandlerException("Error while creating SSHSecurityContext", e,
e.getLocalizedMessage());
+        } catch (GFacException e) {
+            throw new GFacHandlerException("Error while creating SSHSecurityContext", e,
e.getLocalizedMessage());
+        }
     }
 
     public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException,
GFacException {
@@ -205,15 +215,14 @@ public class GSISSHProvider extends AbstractRecoverableProvider {
 
     public void cancelJob(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException
{
         //To change body of implemented methods use File | Settings | File Templates.
-        log.info("cancelling the job status in GSISSHProvider!!!!!");
+        log.info("canceling the job status in GSISSHProvider!!!!!");
         HostDescriptionType host = jobExecutionContext.getApplicationContext().
                 getHostDescription().getType();
         StringBuffer data = new StringBuffer();
         JobDetails jobDetails = jobExecutionContext.getJobDetails();
         try {
             Cluster cluster = null;
-            if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)
!= null) {
-            }else {
+            if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)
== null) {
                 GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
             }
             cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
@@ -249,10 +258,6 @@ public class GSISSHProvider extends AbstractRecoverableProvider {
             GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
             GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT,
ErrorCategory.AIRAVATA_INTERNAL_ERROR);
             throw new GFacProviderException(error, e);
-        } finally {
-            log.info("Saving data for future recovery: ");
-            log.info(data.toString());
-            GFacUtils.savePluginData(jobExecutionContext, data, this.getClass().getName());
         }
     }
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
index 67e6628..4db72a4 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
@@ -61,6 +61,8 @@ import org.slf4j.LoggerFactory;
 
 import sun.reflect.generics.reflectiveObjects.NotImplementedException;
 
+import javax.naming.OperationNotSupportedException;
+
 /**
  * Execute application using remote SSH
  */
@@ -150,7 +152,6 @@ public class SSHProvider extends AbstractProvider {
                 HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
                         getApplicationDeploymentDescription().getType();
                 JobDetails jobDetails = new JobDetails();
-                String taskID = jobExecutionContext.getTaskData().getTaskID();
                 try {
                     Cluster cluster = null;
                     if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)
!= null) {
@@ -204,8 +205,50 @@ public class SSHProvider extends AbstractProvider {
     }
 
 
-    public void cancelJob(JobExecutionContext jobExecutionContext) throws GFacException {
-        throw new NotImplementedException();
+    public void cancelJob(JobExecutionContext jobExecutionContext) throws GFacProviderException,
GFacException {
+        JobDetails jobDetails = jobExecutionContext.getJobDetails();
+        HostDescriptionType host = jobExecutionContext.getApplicationContext().
+                getHostDescription().getType();
+        StringBuffer data = new StringBuffer();
+        if (!hpcType) {
+            throw new NotImplementedException();
+        } else {
+            Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+            if (cluster == null) {
+                throw new GFacProviderException("Security context is not set properly");
+            } else {
+                log.info("Successfully retrieved the Security Context");
+            }
+            // This installed path is a mandetory field, because this could change based
on the computing resource
+            if (jobDetails == null) {
+                log.error("There is not JobDetails so cancelations cannot perform !!!");
+                return;
+            }
+            try {
+                if (jobDetails.getJobID() != null) {
+                    cluster.cancelJob(jobDetails.getJobID());
+                } else {
+                    log.error("No Job Id is set, so cannot perform the cancel operation !!!");
+                    return;
+                }
+                GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.CANCELED);
+            } catch (SSHApiException e) {
+                String error = "Error submitting the job to host " + host.getHostAddress()
+ " message: " + e.getMessage();
+                log.error(error);
+                jobDetails.setJobID("none");
+                GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
+                GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT,
ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+                throw new GFacProviderException(error, e);
+            } catch (Exception e) {
+                String error = "Error submitting the job to host " + host.getHostAddress()
+ " message: " + e.getMessage();
+                log.error(error);
+                jobDetails.setJobID("none");
+                GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
+                GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT,
ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+                throw new GFacProviderException(error, e);
+            }
+            // we know this host is type GsiSSHHostType
+        }
     }
 
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index 8a9764c..7072019 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -43,6 +43,7 @@ import org.apache.airavata.common.utils.AiravataZKUtils;
 import org.apache.airavata.common.utils.Constants;
 import org.apache.airavata.common.utils.ServerSettings;
 import org.apache.airavata.gfac.core.scheduler.HostScheduler;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
 import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
 import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
 import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
@@ -558,6 +559,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
                         taskDetails.setTaskStatus(taskStatus);
                         registry.update(RegistryModelType.TASK_DETAIL, o,
                                 taskDetails);
+                        GFacUtils.setExperimentCancel(experimentId, taskDetails.getTaskID(),
zk);
                     }
                 }
             }else {
@@ -608,6 +610,7 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface,
                             taskDetails.setTaskStatus(taskStatus);
                             registry.update(RegistryModelType.TASK_DETAIL, o,
                                     taskDetails.getTaskID());
+                            GFacUtils.setExperimentCancel(experimentId, taskDetails.getTaskID(),
zk);
                         }
                         // iterate through all the generated tasks and performs the
                         // job submisssion+monitoring

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
index 5f3c263..d57e9a8 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/util/OrchestratorRecoveryHandler.java
@@ -24,6 +24,7 @@ import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.apache.airavata.common.utils.AiravataZKUtils;
 import org.apache.airavata.common.utils.Constants;
 import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
 import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
 import org.apache.airavata.orchestrator.server.OrchestratorServerHandler;
 import org.apache.thrift.TException;
@@ -80,7 +81,11 @@ public class OrchestratorRecoveryHandler implements Watcher {
             log.info("Recovering Experiment: " + expId.split("\\+")[0]);
             log.info("------------------------------------------------------------------------------------");
             try {
-                serverHandler.launchExperiment(expId.split("\\+")[0]);
+                if(GFacUtils.isCancelled(expId.split("\\+")[0], expId.split("\\+")[1], zk))
{// during relaunching we check the operation and then launch
+                    serverHandler.terminateExperiment(expId.split("\\+")[0]);
+                }else {
+                    serverHandler.launchExperiment(expId.split("\\+")[0]);
+                }
                 // we do not move the old experiment in to new gfac node, gfac will do it
             } catch (Exception e) {       // we attempt all the experiments
                 e.printStackTrace();

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
index 89925ee..6c9acec 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/GFACServiceJobSubmitter.java
@@ -149,7 +149,9 @@ public class GFACServiceJobSubmitter implements JobSubmitter, Watcher
{
                 GfacService.Client localhost = GFacClientFactory.createGFacClient(split[0],
Integer.parseInt(split[1]));
                 if (zk.exists(gfacServer + File.separator + pickedChild, false) != null)
{
                     // before submitting the job we check again the state of the node
-                    return localhost.cancelJob(experimentID, taskID);
+                    if (GFacUtils.createExperimentEntry(experimentID, taskID, zk, experimentNode,
pickedChild, null)) {
+                        return localhost.cancelJob(experimentID, taskID);
+                    }
                 }
             }
         } catch (TException e) {

http://git-wip-us.apache.org/repos/asf/airavata/blob/eb9bda87/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
index 713d33d..855c9dc 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
@@ -77,7 +77,7 @@ public class GSISSHAbstractCluster implements Cluster {
         this.jobManagerConfiguration = config;
     }
 
-    public GSISSHAbstractCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo)
throws SSHApiException {
+    public  GSISSHAbstractCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo)
throws SSHApiException {
 
         this.serverInfo = serverInfo;
 


Mime
View raw message