Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9387E11A5C for ; Mon, 5 May 2014 14:44:44 +0000 (UTC) Received: (qmail 10686 invoked by uid 500); 5 May 2014 14:44:36 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 10408 invoked by uid 500); 5 May 2014 14:44:27 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 10320 invoked by uid 99); 5 May 2014 14:44:25 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 05 May 2014 14:44:25 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 28D4490DE61; Mon, 5 May 2014 14:44:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lahiru@apache.org To: commits@airavata.apache.org Date: Mon, 05 May 2014 14:44:27 -0000 Message-Id: <99b886abf32d4bc3a36e7c9e3fa5628c@git.apache.org> In-Reply-To: <1ca351ea5d944d09a6a6033962e8275c@git.apache.org> References: <1ca351ea5d944d09a6a6033962e8275c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/8] changing package names of gfac implementations http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java new file mode 100644 index 0000000..2947fc3 --- /dev/null +++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java @@ -0,0 +1,60 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ + +package org.apache.airavata.gfac.hadoop.provider.utils; + +import org.apache.airavata.gfac.context.JobExecutionContext; +import org.apache.airavata.gfac.context.MessageContext; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.Collection; + +public class HadoopUtils { + public static Configuration createHadoopConfiguration( + JobExecutionContext jobExecutionContext, + boolean isWhirrBasedDeployment, + File hadoopConfigDir) throws FileNotFoundException { + MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); + Configuration hadoopConf = new Configuration(); + + if(isWhirrBasedDeployment){ + hadoopConf.addResource(new FileInputStream( + new File((String)inMessageContext.getParameter("HADOOP_SITE_XML")))); + } else { + readHadoopClusterConfigurationFromDirectory(hadoopConfigDir, hadoopConf); + } + + return hadoopConf; + } + + private static void readHadoopClusterConfigurationFromDirectory(File localHadoopConfigurationDirectory, Configuration hadoopConf) + throws FileNotFoundException { + Collection hadoopConfigurationFiles = + FileUtils.listFiles(localHadoopConfigurationDirectory, null, false); + for (Object f : hadoopConfigurationFiles) { + hadoopConf.addResource(new FileInputStream((File)f)); + } + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java deleted file mode 100644 index 49eadf4..0000000 --- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ - -package org.apache.airavata.gfac.handler; - -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.gfac.context.MessageContext; -import org.apache.airavata.gfac.provider.utils.HadoopUtils; -import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; -import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.Map; - -public class HDFSDataMovementHandler implements GFacHandler { - private static final Logger logger = LoggerFactory.getLogger(HDFSDataMovementHandler.class); - - private boolean isWhirrBasedDeployment = false; - private File hadoopConfigDir; - - public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException { - MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); - if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){ - isWhirrBasedDeployment = true; - } else { - String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR"); - File hadoopConfigDir = new File(hadoopConfigDirPath); - if (!hadoopConfigDir.exists()){ - throw new GFacHandlerException("Specified hadoop configuration directory doesn't exist."); - } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){ - throw new GFacHandlerException("Cannot find any hadoop configuration files inside specified directory."); - } - - this.hadoopConfigDir = hadoopConfigDir; - } - - if(jobExecutionContext.isInPath()){ - try { - handleInPath(jobExecutionContext); - } catch (IOException e) { - throw new GFacHandlerException("Error while copying input data from local file system to HDFS.",e); - } - } else { - handleOutPath(jobExecutionContext); - } - } - - private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException, IOException { - ApplicationDeploymentDescriptionType appDepDesc = - jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType(); - HadoopApplicationDeploymentDescriptionType hadoopAppDesc = - (HadoopApplicationDeploymentDescriptionType)appDepDesc; - if(appDepDesc.isSetInputDataDirectory() && isInputDataDirectoryLocal(appDepDesc)){ - Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir); - FileSystem hdfs = FileSystem.get(hadoopConf); - hdfs.copyFromLocalFile(new Path(appDepDesc.getInputDataDirectory()), - new Path(hadoopAppDesc.getHadoopJobConfiguration().getHdfsInputDirectory())); - } - } - - private boolean isInputDataDirectoryLocal(ApplicationDeploymentDescriptionType appDepDesc){ - String inputDataDirectoryPath = appDepDesc.getInputDataDirectory(); - File inputDataDirectory = new File(inputDataDirectoryPath); - if(inputDataDirectory.exists() && FileUtils.listFiles(inputDataDirectory, null, null).size() > 0){ - return true; - } - - return false; - } - - private void handleOutPath(JobExecutionContext jobExecutionContext){} - - public void initProperties(Map properties) throws GFacHandlerException { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java deleted file mode 100644 index c7de3f9..0000000 --- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ - -package org.apache.airavata.gfac.handler; - -import com.google.common.io.Files; -import org.apache.airavata.commons.gfac.type.HostDescription; -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.gfac.context.MessageContext; -import org.apache.airavata.schemas.gfac.HadoopHostType; -import org.apache.commons.configuration.CompositeConfiguration; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.io.FileUtils; -import org.apache.whirr.Cluster; -import org.apache.whirr.ClusterController; -import org.apache.whirr.ClusterControllerFactory; -import org.apache.whirr.ClusterSpec; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.*; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.EnumSet; -import java.util.Map; -import java.util.Properties; - -import static org.apache.whirr.ClusterSpec.Property.*; -import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES; -import static org.apache.whirr.ClusterSpec.Property.PRIVATE_KEY_FILE; - -/** - * This handler takes care of deploying hadoop in cloud(in cloud bursting scenarios) and - * deploying hadoop in local cluster. In case of existing hadoop cluster this will ignore - * cluster setup just use the hadoop configuration provided by user. - */ -public class HadoopDeploymentHandler implements GFacHandler { - private static final Logger logger = LoggerFactory.getLogger("hadoop-dep-handler"); - - /** - * Once invoked this method will deploy Hadoop in a local cluster or cloud based on the - * configuration provided. If there is a already deployed hadoop cluster this will skip - * deployment. - * - * - * @param jobExecutionContext job execution context containing all the required configurations - * and runtime information. - * @throws GFacHandlerException - */ - public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException { - if(jobExecutionContext.isInPath()){ - handleInPath(jobExecutionContext); - } else { - handleOutPath(jobExecutionContext); - } - } - - private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException { - HostDescription hostDescription = - jobExecutionContext.getApplicationContext().getHostDescription(); - if (!isHadoopDeploymentAvailable(hostDescription)) { - // Temp directory to keep generated configuration files. - File tempDirectory = Files.createTempDir(); - try { - File hadoopSiteXML = launchHadoopCluster(hostDescription, tempDirectory); - jobExecutionContext.getInMessageContext().addParameter("HADOOP_SITE_XML", hadoopSiteXML.getAbsolutePath()); - jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE", "WHIRR"); - // TODO: Add hadoop-site.xml to job execution context. - } catch (IOException e) { - throw new GFacHandlerException("IO Error while processing configurations.",e); - } catch (ConfigurationException e) { - throw new GFacHandlerException("Whirr configuration error.", e); - } catch (InterruptedException e) { - throw new GFacHandlerException("Hadoop cluster launch interrupted.", e); - } catch (TransformerException e) { - throw new GFacHandlerException("Error while creating hadoop-site.xml", e); - } catch (ParserConfigurationException e) { - throw new GFacHandlerException("Error while creating hadoop-site.xml", e); - } - } else { - jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE", - "MANUAL"); - jobExecutionContext.getInMessageContext().addParameter("HADOOP_CONFIG_DIR", - ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory()); - logger.info("Hadoop configuration is available. Skipping hadoop deployment."); - if(logger.isDebugEnabled()){ - logger.debug("Hadoop configuration directory: " + - getHadoopConfigDirectory(hostDescription)); - } - } - } - - private void handleOutPath(JobExecutionContext jobExecutionContext){ - MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); - if(((String)inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE")).equals("WHIRR")){ - // TODO: Shutdown hadoop cluster. - logger.info("Shutdown hadoop cluster."); - } - } - - private File launchHadoopCluster(HostDescription hostDescription, File workingDirectory) - throws IOException, GFacHandlerException, ConfigurationException, InterruptedException, TransformerException, ParserConfigurationException { - ClusterSpec hadoopClusterSpec = - whirrConfigurationToClusterSpec(hostDescription, workingDirectory); - ClusterController hadoopClusterController = - createClusterController(hadoopClusterSpec.getServiceName()); - Cluster hadoopCluster = hadoopClusterController.launchCluster(hadoopClusterSpec); - - logger.info(String.format("Started cluster of %s instances.\n", - hadoopCluster.getInstances().size())); - - File siteXML = new File(workingDirectory, "hadoop-site.xml"); - clusterPropertiesToHadoopSiteXml(hadoopCluster.getConfiguration(), siteXML); - - return siteXML; - } - - private ClusterController createClusterController(String serviceName){ - ClusterControllerFactory factory = new ClusterControllerFactory(); - ClusterController controller = factory.create(serviceName); - - if(controller == null){ - logger.warn("Unable to find the service {0}, using default.", serviceName); - controller = factory.create(null); - } - - return controller; - } - - private ClusterSpec whirrConfigurationToClusterSpec(HostDescription hostDescription, - File workingDirectory) throws IOException, GFacHandlerException, ConfigurationException { - File whirrConfig = getWhirrConfigurationFile(hostDescription, workingDirectory); - CompositeConfiguration compositeConfiguration = new CompositeConfiguration(); - Configuration configuration = new PropertiesConfiguration(whirrConfig); - compositeConfiguration.addConfiguration(configuration); - - ClusterSpec hadoopClusterSpec = new ClusterSpec(compositeConfiguration); - - for (ClusterSpec.Property required : EnumSet.of(CLUSTER_NAME, PROVIDER, IDENTITY, CREDENTIAL, - INSTANCE_TEMPLATES, PRIVATE_KEY_FILE)) { - if (hadoopClusterSpec.getConfiguration().getString(required.getConfigName()) == null) { - throw new IllegalArgumentException(String.format("Option '%s' not set.", - required.getSimpleName())); - } - } - - return hadoopClusterSpec; - } - - private File getWhirrConfigurationFile(HostDescription hostDescription, File workingDirectory) - throws GFacHandlerException, IOException { - HadoopHostType hadoopHostDesc = (HadoopHostType)hostDescription; - if(hadoopHostDesc.isSetWhirrConfiguration()){ - HadoopHostType.WhirrConfiguration whirrConfig = hadoopHostDesc.getWhirrConfiguration(); - if(whirrConfig.isSetConfigurationFile()){ - File whirrConfigFile = new File(whirrConfig.getConfigurationFile()); - if(!whirrConfigFile.exists()){ - throw new GFacHandlerException( - "Specified whirr configuration file doesn't exists."); - } - - FileUtils.copyFileToDirectory(whirrConfigFile, workingDirectory); - - return new File(workingDirectory, whirrConfigFile.getName()); - } else if(whirrConfig.isSetConfiguration()){ - Properties whirrConfigProps = - whirrConfigurationsToProperties(whirrConfig.getConfiguration()); - File whirrConfigFile = new File(workingDirectory, "whirr-hadoop.config"); - whirrConfigProps.store( - new FileOutputStream(whirrConfigFile), null); - - return whirrConfigFile; - } - } - - throw new GFacHandlerException("Cannot find Whirr configurations. Whirr configuration " - + "is required if you don't have already running Hadoop deployment."); - } - - private Properties whirrConfigurationsToProperties( - HadoopHostType.WhirrConfiguration.Configuration configuration){ - Properties whirrConfigProps = new Properties(); - - for(HadoopHostType.WhirrConfiguration.Configuration.Property property: - configuration.getPropertyArray()) { - whirrConfigProps.put(property.getName(), property.getValue()); - } - - return whirrConfigProps; - } - - private void clusterPropertiesToHadoopSiteXml(Properties props, File hadoopSiteXml) throws ParserConfigurationException, TransformerException { - DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder documentBuilder = domFactory.newDocumentBuilder(); - - Document hadoopSiteXmlDoc = documentBuilder.newDocument(); - - hadoopSiteXmlDoc.setXmlVersion("1.0"); - hadoopSiteXmlDoc.setXmlStandalone(true); - hadoopSiteXmlDoc.createProcessingInstruction("xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\""); - - Element configEle = hadoopSiteXmlDoc.createElement("configuration"); - - hadoopSiteXmlDoc.appendChild(configEle); - - for(Map.Entry entry : props.entrySet()){ - addPropertyToConfiguration(entry, configEle, hadoopSiteXmlDoc); - } - - saveDomToFile(hadoopSiteXmlDoc, hadoopSiteXml); - } - - private void saveDomToFile(Document dom, File destFile) throws TransformerException { - Source source = new DOMSource(dom); - - Result result = new StreamResult(destFile); - - Transformer transformer = TransformerFactory.newInstance().newTransformer(); - transformer.transform(source, result); - } - - private void addPropertyToConfiguration(Map.Entry entry, Element configElement, Document doc){ - Element property = doc.createElement("property"); - configElement.appendChild(property); - - Element nameEle = doc.createElement("name"); - nameEle.setTextContent(entry.getKey().toString()); - property.appendChild(nameEle); - - Element valueEle = doc.createElement("value"); - valueEle.setTextContent(entry.getValue().toString()); - property.appendChild(valueEle); - } - - private boolean isHadoopDeploymentAvailable(HostDescription hostDescription) { - return ((HadoopHostType) hostDescription.getType()).isSetHadoopConfigurationDirectory(); - } - - private String getHadoopConfigDirectory(HostDescription hostDescription){ - return ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory(); - } - - public void initProperties(Map properties) throws GFacHandlerException { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java deleted file mode 100644 index 0fc8b47..0000000 --- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ - -package org.apache.airavata.gfac.provider.impl; - -import java.io.File; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Map; - -import org.apache.airavata.commons.gfac.type.ActualParameter; -import org.apache.airavata.gfac.GFacException; -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.gfac.context.MessageContext; -import org.apache.airavata.gfac.provider.AbstractProvider; -import org.apache.airavata.gfac.provider.GFacProviderException; -import org.apache.airavata.gfac.provider.utils.HadoopUtils; -import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType; -import org.apache.airavata.schemas.gfac.OutputParameterType; -import org.apache.airavata.schemas.gfac.StringParameterType; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.OutputFormat; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -/** - * Executes hadoop job using the cluster configuration provided by handlers in - * in-flow. - */ -public class HadoopProvider extends AbstractProvider { - private static final Logger logger = LoggerFactory.getLogger(HadoopProvider.class); - - private boolean isWhirrBasedDeployment = false; - private File hadoopConfigDir; - - public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException { - MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); - if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){ - isWhirrBasedDeployment = true; - } else { - String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR"); - File hadoopConfigDir = new File(hadoopConfigDirPath); - if (!hadoopConfigDir.exists()){ - throw new GFacProviderException("Specified hadoop configuration directory doesn't exist."); - } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){ - throw new GFacProviderException("Cannot find any hadoop configuration files inside specified directory."); - } - - this.hadoopConfigDir = hadoopConfigDir; - } - } - - public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException { - HadoopApplicationDeploymentDescriptionType hadoopAppDesc = - (HadoopApplicationDeploymentDescriptionType)jobExecutionContext - .getApplicationContext().getApplicationDeploymentDescription().getType(); - MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); - HadoopApplicationDeploymentDescriptionType.HadoopJobConfiguration jobConf = hadoopAppDesc.getHadoopJobConfiguration(); - - try{ - // Preparing Hadoop configuration - Configuration hadoopConf = HadoopUtils.createHadoopConfiguration( - jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir); - - // Load jar containing map-reduce job implementation - ArrayList mapRedJars = new ArrayList(); - mapRedJars.add(new File(jobConf.getJarLocation()).toURL()); - URLClassLoader childClassLoader = new URLClassLoader(mapRedJars.toArray(new URL[mapRedJars.size()]), - this.getClass().getClassLoader()); - - Job job = new Job(hadoopConf); - - job.setJobName(jobConf.getJobName()); - - job.setOutputKeyClass(Class.forName(jobConf.getOutputKeyClass(), true, childClassLoader)); - job.setOutputValueClass(Class.forName(jobConf.getOutputValueClass(), true, childClassLoader)); - - job.setMapperClass((Class)Class.forName(jobConf.getMapperClass(), true, childClassLoader)); - job.setCombinerClass((Class) Class.forName(jobConf.getCombinerClass(), true, childClassLoader)); - job.setReducerClass((Class) Class.forName(jobConf.getCombinerClass(), true, childClassLoader)); - - job.setInputFormatClass((Class)Class.forName(jobConf.getInputFormatClass(), true, childClassLoader)); - job.setOutputFormatClass((Class) Class.forName(jobConf.getOutputFormatClass(), true, childClassLoader)); - - FileInputFormat.setInputPaths(job, new Path(hadoopAppDesc.getInputDataDirectory())); - FileOutputFormat.setOutputPath(job, new Path(hadoopAppDesc.getOutputDataDirectory())); - - job.waitForCompletion(true); - System.out.println(job.getTrackingURL()); - if(jobExecutionContext.getOutMessageContext() == null){ - jobExecutionContext.setOutMessageContext(new MessageContext()); - } - - OutputParameterType[] outputParametersArray = jobExecutionContext.getApplicationContext(). - getServiceDescription().getType().getOutputParametersArray(); - for(OutputParameterType outparamType : outputParametersArray){ - String paramName = outparamType.getParameterName(); - if(paramName.equals("test-hadoop")){ - ActualParameter outParam = new ActualParameter(); - outParam.getType().changeType(StringParameterType.type); - ((StringParameterType) outParam.getType()).setValue(job.getTrackingURL()); - jobExecutionContext.getOutMessageContext().addParameter("test-hadoop", outParam); - } - } - } catch (Exception e) { - String errMessage = "Error occurred during Map-Reduce job execution."; - logger.error(errMessage, e); - throw new GFacProviderException(errMessage, e); - } - } - - public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException { - // TODO: How to handle cluster shutdown. Best way is to introduce inPath/outPath to handler. - } - - @Override - public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException { - throw new NotImplementedException(); - } - - - public void initProperties(Map properties) throws GFacProviderException, GFacException { - - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java deleted file mode 100644 index c3053d1..0000000 --- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ - -package org.apache.airavata.gfac.provider.utils; - -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.gfac.context.MessageContext; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.util.Collection; - -public class HadoopUtils { - public static Configuration createHadoopConfiguration( - JobExecutionContext jobExecutionContext, - boolean isWhirrBasedDeployment, - File hadoopConfigDir) throws FileNotFoundException { - MessageContext inMessageContext = jobExecutionContext.getInMessageContext(); - Configuration hadoopConf = new Configuration(); - - if(isWhirrBasedDeployment){ - hadoopConf.addResource(new FileInputStream( - new File((String)inMessageContext.getParameter("HADOOP_SITE_XML")))); - } else { - readHadoopClusterConfigurationFromDirectory(hadoopConfigDir, hadoopConf); - } - - return hadoopConf; - } - - private static void readHadoopClusterConfigurationFromDirectory(File localHadoopConfigurationDirectory, Configuration hadoopConf) - throws FileNotFoundException { - Collection hadoopConfigurationFiles = - FileUtils.listFiles(localHadoopConfigurationDirectory, null, false); - for (Object f : hadoopConfigurationFiles) { - hadoopConf.addResource(new FileInputStream((File)f)); - } - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml index f43a5fb..9c4774c 100644 --- a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml +++ b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml @@ -20,21 +20,11 @@ - + - - + + - - - - - - - - - - \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java deleted file mode 100644 index 8a11c2a..0000000 --- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.gfac.handler; - -import org.apache.airavata.commons.gfac.type.ApplicationDescription; -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; -import org.apache.airavata.schemas.gfac.HostDescriptionType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Map; - -public class LocalDirectorySetupHandler implements GFacHandler{ - private static final Logger log = LoggerFactory.getLogger(LocalDirectorySetupHandler.class); - - public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException { - log.info("Invoking GramDirectorySetupHandler ..."); - HostDescriptionType type = jobExecutionContext.getApplicationContext().getHostDescription().getType(); - ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); - ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType(); - log.debug("working diectroy = " + app.getStaticWorkingDirectory()); - log.debug("temp directory = " + app.getScratchWorkingDirectory()); - - makeFileSystemDir(app.getStaticWorkingDirectory(),jobExecutionContext); - makeFileSystemDir(app.getScratchWorkingDirectory(),jobExecutionContext); - makeFileSystemDir(app.getInputDataDirectory(),jobExecutionContext); - makeFileSystemDir(app.getOutputDataDirectory(),jobExecutionContext); - } - private void makeFileSystemDir(String dir, JobExecutionContext jobExecutionContext) throws GFacHandlerException { - File f = new File(dir); - if (f.isDirectory() && f.exists()) { - return; - } else if (!new File(dir).mkdir()) { - throw new GFacHandlerException("Cannot make directory "+dir); - } - } - - public void initProperties(Map properties) throws GFacHandlerException { - - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java new file mode 100644 index 0000000..81493fb --- /dev/null +++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java @@ -0,0 +1,63 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.local.handler; + +import org.apache.airavata.commons.gfac.type.ApplicationDescription; +import org.apache.airavata.gfac.context.JobExecutionContext; +import org.apache.airavata.gfac.handler.GFacHandler; +import org.apache.airavata.gfac.handler.GFacHandlerException; +import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; +import org.apache.airavata.schemas.gfac.HostDescriptionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Map; + +public class LocalDirectorySetupHandler implements GFacHandler { + private static final Logger log = LoggerFactory.getLogger(LocalDirectorySetupHandler.class); + + public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException { + log.info("Invoking GramDirectorySetupHandler ..."); + HostDescriptionType type = jobExecutionContext.getApplicationContext().getHostDescription().getType(); + ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); + ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType(); + log.debug("working diectroy = " + app.getStaticWorkingDirectory()); + log.debug("temp directory = " + app.getScratchWorkingDirectory()); + + makeFileSystemDir(app.getStaticWorkingDirectory(),jobExecutionContext); + makeFileSystemDir(app.getScratchWorkingDirectory(),jobExecutionContext); + makeFileSystemDir(app.getInputDataDirectory(),jobExecutionContext); + makeFileSystemDir(app.getOutputDataDirectory(),jobExecutionContext); + } + private void makeFileSystemDir(String dir, JobExecutionContext jobExecutionContext) throws GFacHandlerException { + File f = new File(dir); + if (f.isDirectory() && f.exists()) { + return; + } else if (!new File(dir).mkdir()) { + throw new GFacHandlerException("Cannot make directory "+dir); + } + } + + public void initProperties(Map properties) throws GFacHandlerException { + + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java new file mode 100644 index 0000000..f0a0bf9 --- /dev/null +++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java @@ -0,0 +1,239 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.local.impl; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.airavata.gfac.Constants; +import org.apache.airavata.gfac.GFacException; +import org.apache.airavata.gfac.context.JobExecutionContext; +import org.apache.airavata.gfac.local.utils.InputStreamToFileWriter; +import org.apache.airavata.gfac.local.utils.InputUtils; +import org.apache.airavata.gfac.notification.events.StartExecutionEvent; +import org.apache.airavata.gfac.provider.AbstractProvider; +import org.apache.airavata.gfac.provider.GFacProviderException; +import org.apache.airavata.gfac.provider.utils.ProviderUtils; +import org.apache.airavata.gfac.utils.GFacUtils; +import org.apache.airavata.gfac.utils.OutputUtils; +import org.apache.airavata.model.workspace.experiment.JobDetails; +import org.apache.airavata.model.workspace.experiment.JobState; +import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; +import org.apache.airavata.schemas.gfac.NameValuePairType; +import org.apache.xmlbeans.XmlException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +public class LocalProvider extends AbstractProvider { + private static final Logger log = LoggerFactory.getLogger(LocalProvider.class); + private ProcessBuilder builder; + private List cmdList; + private String jobId; + + public static class LocalProviderJobData{ + private String applicationName; + private List inputParameters; + private String workingDir; + private String inputDir; + private String outputDir; + public String getApplicationName() { + return applicationName; + } + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + public List getInputParameters() { + return inputParameters; + } + public void setInputParameters(List inputParameters) { + this.inputParameters = inputParameters; + } + public String getWorkingDir() { + return workingDir; + } + public void setWorkingDir(String workingDir) { + this.workingDir = workingDir; + } + public String getInputDir() { + return inputDir; + } + public void setInputDir(String inputDir) { + this.inputDir = inputDir; + } + public String getOutputDir() { + return outputDir; + } + public void setOutputDir(String outputDir) { + this.outputDir = outputDir; + } + } + public LocalProvider(){ + cmdList = new ArrayList(); + } + + public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException { + super.initialize(jobExecutionContext); + ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext(). + getApplicationDeploymentDescription().getType(); + + buildCommand(app.getExecutableLocation(), ProviderUtils.getInputParameters(jobExecutionContext)); + initProcessBuilder(app); + + // extra environment variables + builder.environment().put(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory()); + builder.environment().put(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory()); + + // set working directory + builder.directory(new File(app.getStaticWorkingDirectory())); + + // log info + log.info("Command = " + InputUtils.buildCommand(cmdList)); + log.info("Working dir = " + builder.directory()); + for (String key : builder.environment().keySet()) { + log.info("Env[" + key + "] = " + builder.environment().get(key)); + } + } + + public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException { + jobExecutionContext.getNotifier().publish(new StartExecutionEvent()); + ApplicationDeploymentDescriptionType app = jobExecutionContext. + getApplicationContext().getApplicationDeploymentDescription().getType(); + JobDetails jobDetails = new JobDetails(); + try { + jobId = jobExecutionContext.getTaskData().getTaskID(); + jobDetails.setJobID(jobId); + jobDetails.setJobDescription(app.toString()); + jobExecutionContext.setJobDetails(jobDetails); + jobDetails.setJobDescription(app.toString()); + GFacUtils.saveJobStatus(jobExecutionContext,jobDetails, JobState.SETUP); + // running cmd + Process process = builder.start(); + + Thread standardOutWriter = new InputStreamToFileWriter(process.getInputStream(), app.getStandardOutput()); + Thread standardErrorWriter = new InputStreamToFileWriter(process.getErrorStream(), app.getStandardError()); + + // start output threads + standardOutWriter.setDaemon(true); + standardErrorWriter.setDaemon(true); + standardOutWriter.start(); + standardErrorWriter.start(); + + int returnValue = process.waitFor(); + + // make sure other two threads are done + standardOutWriter.join(); + standardErrorWriter.join(); + + /* + * check return value. usually not very helpful to draw conclusions based on return values so don't bother. + * just provide warning in the log messages + */ + if (returnValue != 0) { + log.error("Process finished with non zero return value. Process may have failed"); + } else { + log.info("Process finished with return value of zero."); + } + + StringBuffer buf = new StringBuffer(); + buf.append("Executed ").append(InputUtils.buildCommand(cmdList)) + .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory()) + .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ") + .append(String.valueOf(returnValue)); + log.info(buf.toString()); + } catch (IOException io) { + throw new GFacProviderException(io.getMessage(), io); + } catch (InterruptedException e) { + throw new GFacProviderException(e.getMessage(), e); + }catch (GFacException e) { + throw new GFacProviderException(e.getMessage(), e); + } + } + +// private void saveApplicationJob(JobExecutionContext jobExecutionContext) +// throws GFacProviderException { +// ApplicationDeploymentDescriptionType app = jobExecutionContext. +// getApplicationContext().getApplicationDeploymentDescription().getType(); +// ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext); +// appJob.setJobId(jobId); +// LocalProviderJobData data = new LocalProviderJobData(); +// data.setApplicationName(app.getExecutableLocation()); +// data.setInputDir(app.getInputDataDirectory()); +// data.setOutputDir(app.getOutputDataDirectory()); +// data.setWorkingDir(builder.directory().toString()); +// data.setInputParameters(ProviderUtils.getInputParameters(jobExecutionContext)); +// ByteArrayOutputStream stream = new ByteArrayOutputStream(); +// JAXB.marshal(data, stream); +// appJob.setJobData(stream.toString()); +// appJob.setSubmittedTime(Calendar.getInstance().getTime()); +// appJob.setStatus(ApplicationJobStatus.SUBMITTED); +// appJob.setStatusUpdateTime(appJob.getSubmittedTime()); +// GFacUtils.recordApplicationJob(jobExecutionContext, appJob); +// } + + public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException { + ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType(); + + try { + String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput()); + String stdErrStr = GFacUtils.readFileToString(app.getStandardError()); + Map output = jobExecutionContext.getOutMessageContext().getParameters(); + OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + } catch (XmlException e) { + throw new GFacProviderException("Cannot read output:" + e.getMessage(), e); + } catch (IOException io) { + throw new GFacProviderException(io.getMessage(), io); + } catch (Exception e){ + throw new GFacProviderException("Error in retrieving results",e); + } + } + + public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException { + throw new NotImplementedException(); + } + + + private void buildCommand(String executable, List inputParameterList){ + cmdList.add(executable); + cmdList.addAll(inputParameterList); + } + + private void initProcessBuilder(ApplicationDeploymentDescriptionType app){ + builder = new ProcessBuilder(cmdList); + + NameValuePairType[] env = app.getApplicationEnvironmentArray(); + + if(env != null && env.length > 0){ + Map builderEnv = builder.environment(); + for (NameValuePairType entry : env) { + builderEnv.put(entry.getName(), entry.getValue()); + } + } + } + + public void initProperties(Map properties) throws GFacProviderException, GFacException { + + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java new file mode 100644 index 0000000..2c9a4d2 --- /dev/null +++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java @@ -0,0 +1,68 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.local.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; + +public class InputStreamToFileWriter extends Thread{ + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + private BufferedReader in; + private BufferedWriter out; + + public InputStreamToFileWriter(InputStream in, String out) throws IOException { + this.in = new BufferedReader(new InputStreamReader(in)); + this.out = new BufferedWriter(new FileWriter(out)); + } + + public void run() { + try { + String line = null; + while ((line = in.readLine()) != null) { + if (log.isDebugEnabled()) { + log.debug(line); + } + out.write(line); + out.newLine(); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (in != null) { + try { + in.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + if (out != null) { + try { + out.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java new file mode 100644 index 0000000..ba65753 --- /dev/null +++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java @@ -0,0 +1,39 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.local.utils; + +import java.util.List; + +public class InputUtils { + private static final String SPACE = " "; + + private InputUtils() { + } + + public static String buildCommand(List cmdList) { + StringBuffer buff = new StringBuffer(); + for (String string : cmdList) { + buff.append(string); + buff.append(SPACE); + } + return buff.toString(); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java new file mode 100644 index 0000000..4a19b65 --- /dev/null +++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java @@ -0,0 +1,54 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.local.utils; + +import org.apache.airavata.gfac.context.JobExecutionContext; +import org.apache.airavata.gfac.provider.GFacProviderException; +import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +public class LocalProviderUtil { + private static final Logger log = LoggerFactory.getLogger(LocalProviderUtil.class); + + private void makeFileSystemDir(String dir) throws GFacProviderException { + File f = new File(dir); + if (f.isDirectory() && f.exists()) { + return; + } else if (!new File(dir).mkdir()) { + throw new GFacProviderException("Cannot make directory " + dir); + } + } + + public void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacProviderException { + ApplicationDeploymentDescriptionType app = jobExecutionContext. + getApplicationContext().getApplicationDeploymentDescription().getType(); + log.info("working diectroy = " + app.getStaticWorkingDirectory()); + log.info("temp directory = " + app.getScratchWorkingDirectory()); + makeFileSystemDir(app.getStaticWorkingDirectory()); + makeFileSystemDir(app.getScratchWorkingDirectory()); + makeFileSystemDir(app.getInputDataDirectory()); + makeFileSystemDir(app.getOutputDataDirectory()); + } + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java deleted file mode 100644 index 6a92e53..0000000 --- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.gfac.provider.impl; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.airavata.gfac.Constants; -import org.apache.airavata.gfac.GFacException; -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.gfac.notification.events.StartExecutionEvent; -import org.apache.airavata.gfac.provider.AbstractProvider; -import org.apache.airavata.gfac.provider.GFacProviderException; -import org.apache.airavata.gfac.provider.utils.ProviderUtils; -import org.apache.airavata.gfac.utils.GFacUtils; -import org.apache.airavata.gfac.utils.InputStreamToFileWriter; -import org.apache.airavata.gfac.utils.InputUtils; -import org.apache.airavata.gfac.utils.OutputUtils; -import org.apache.airavata.model.workspace.experiment.JobDetails; -import org.apache.airavata.model.workspace.experiment.JobState; -import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; -import org.apache.airavata.schemas.gfac.NameValuePairType; -import org.apache.xmlbeans.XmlException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -public class LocalProvider extends AbstractProvider { - private static final Logger log = LoggerFactory.getLogger(LocalProvider.class); - private ProcessBuilder builder; - private List cmdList; - private String jobId; - - public static class LocalProviderJobData{ - private String applicationName; - private List inputParameters; - private String workingDir; - private String inputDir; - private String outputDir; - public String getApplicationName() { - return applicationName; - } - public void setApplicationName(String applicationName) { - this.applicationName = applicationName; - } - public List getInputParameters() { - return inputParameters; - } - public void setInputParameters(List inputParameters) { - this.inputParameters = inputParameters; - } - public String getWorkingDir() { - return workingDir; - } - public void setWorkingDir(String workingDir) { - this.workingDir = workingDir; - } - public String getInputDir() { - return inputDir; - } - public void setInputDir(String inputDir) { - this.inputDir = inputDir; - } - public String getOutputDir() { - return outputDir; - } - public void setOutputDir(String outputDir) { - this.outputDir = outputDir; - } - } - public LocalProvider(){ - cmdList = new ArrayList(); - } - - public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException { - super.initialize(jobExecutionContext); - ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext(). - getApplicationDeploymentDescription().getType(); - - buildCommand(app.getExecutableLocation(), ProviderUtils.getInputParameters(jobExecutionContext)); - initProcessBuilder(app); - - // extra environment variables - builder.environment().put(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory()); - builder.environment().put(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory()); - - // set working directory - builder.directory(new File(app.getStaticWorkingDirectory())); - - // log info - log.info("Command = " + InputUtils.buildCommand(cmdList)); - log.info("Working dir = " + builder.directory()); - for (String key : builder.environment().keySet()) { - log.info("Env[" + key + "] = " + builder.environment().get(key)); - } - } - - public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException { - jobExecutionContext.getNotifier().publish(new StartExecutionEvent()); - ApplicationDeploymentDescriptionType app = jobExecutionContext. - getApplicationContext().getApplicationDeploymentDescription().getType(); - JobDetails jobDetails = new JobDetails(); - try { - jobId = jobExecutionContext.getTaskData().getTaskID(); - jobDetails.setJobID(jobId); - jobDetails.setJobDescription(app.toString()); - jobExecutionContext.setJobDetails(jobDetails); - jobDetails.setJobDescription(app.toString()); - GFacUtils.saveJobStatus(jobExecutionContext,jobDetails, JobState.SETUP); - // running cmd - Process process = builder.start(); - - Thread standardOutWriter = new InputStreamToFileWriter(process.getInputStream(), app.getStandardOutput()); - Thread standardErrorWriter = new InputStreamToFileWriter(process.getErrorStream(), app.getStandardError()); - - // start output threads - standardOutWriter.setDaemon(true); - standardErrorWriter.setDaemon(true); - standardOutWriter.start(); - standardErrorWriter.start(); - - int returnValue = process.waitFor(); - - // make sure other two threads are done - standardOutWriter.join(); - standardErrorWriter.join(); - - /* - * check return value. usually not very helpful to draw conclusions based on return values so don't bother. - * just provide warning in the log messages - */ - if (returnValue != 0) { - log.error("Process finished with non zero return value. Process may have failed"); - } else { - log.info("Process finished with return value of zero."); - } - - StringBuffer buf = new StringBuffer(); - buf.append("Executed ").append(InputUtils.buildCommand(cmdList)) - .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory()) - .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ") - .append(String.valueOf(returnValue)); - log.info(buf.toString()); - } catch (IOException io) { - throw new GFacProviderException(io.getMessage(), io); - } catch (InterruptedException e) { - throw new GFacProviderException(e.getMessage(), e); - }catch (GFacException e) { - throw new GFacProviderException(e.getMessage(), e); - } - } - -// private void saveApplicationJob(JobExecutionContext jobExecutionContext) -// throws GFacProviderException { -// ApplicationDeploymentDescriptionType app = jobExecutionContext. -// getApplicationContext().getApplicationDeploymentDescription().getType(); -// ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext); -// appJob.setJobId(jobId); -// LocalProviderJobData data = new LocalProviderJobData(); -// data.setApplicationName(app.getExecutableLocation()); -// data.setInputDir(app.getInputDataDirectory()); -// data.setOutputDir(app.getOutputDataDirectory()); -// data.setWorkingDir(builder.directory().toString()); -// data.setInputParameters(ProviderUtils.getInputParameters(jobExecutionContext)); -// ByteArrayOutputStream stream = new ByteArrayOutputStream(); -// JAXB.marshal(data, stream); -// appJob.setJobData(stream.toString()); -// appJob.setSubmittedTime(Calendar.getInstance().getTime()); -// appJob.setStatus(ApplicationJobStatus.SUBMITTED); -// appJob.setStatusUpdateTime(appJob.getSubmittedTime()); -// GFacUtils.recordApplicationJob(jobExecutionContext, appJob); -// } - - public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException { - ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType(); - - try { - String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput()); - String stdErrStr = GFacUtils.readFileToString(app.getStandardError()); - Map output = jobExecutionContext.getOutMessageContext().getParameters(); - OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); - } catch (XmlException e) { - throw new GFacProviderException("Cannot read output:" + e.getMessage(), e); - } catch (IOException io) { - throw new GFacProviderException(io.getMessage(), io); - } catch (Exception e){ - throw new GFacProviderException("Error in retrieving results",e); - } - } - - public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException { - throw new NotImplementedException(); - } - - - private void buildCommand(String executable, List inputParameterList){ - cmdList.add(executable); - cmdList.addAll(inputParameterList); - } - - private void initProcessBuilder(ApplicationDeploymentDescriptionType app){ - builder = new ProcessBuilder(cmdList); - - NameValuePairType[] env = app.getApplicationEnvironmentArray(); - - if(env != null && env.length > 0){ - Map builderEnv = builder.environment(); - for (NameValuePairType entry : env) { - builderEnv.put(entry.getName(), entry.getValue()); - } - } - } - - public void initProperties(Map properties) throws GFacProviderException, GFacException { - - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java deleted file mode 100644 index 691e11d..0000000 --- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.gfac.utils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; - -public class InputStreamToFileWriter extends Thread{ - protected final Logger log = LoggerFactory.getLogger(this.getClass()); - - private BufferedReader in; - private BufferedWriter out; - - public InputStreamToFileWriter(InputStream in, String out) throws IOException { - this.in = new BufferedReader(new InputStreamReader(in)); - this.out = new BufferedWriter(new FileWriter(out)); - } - - public void run() { - try { - String line = null; - while ((line = in.readLine()) != null) { - if (log.isDebugEnabled()) { - log.debug(line); - } - out.write(line); - out.newLine(); - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (in != null) { - try { - in.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - if (out != null) { - try { - out.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java deleted file mode 100644 index dedfa29..0000000 --- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.gfac.utils; - -import java.util.List; - -public class InputUtils { - private static final String SPACE = " "; - - private InputUtils() { - } - - public static String buildCommand(List cmdList) { - StringBuffer buff = new StringBuffer(); - for (String string : cmdList) { - buff.append(string); - buff.append(SPACE); - } - return buff.toString(); - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java deleted file mode 100644 index c80eeda..0000000 --- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.gfac.utils; - -import org.apache.airavata.gfac.context.JobExecutionContext; -import org.apache.airavata.gfac.provider.GFacProviderException; -import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; - -public class LocalProviderUtil { - private static final Logger log = LoggerFactory.getLogger(LocalProviderUtil.class); - - private void makeFileSystemDir(String dir) throws GFacProviderException { - File f = new File(dir); - if (f.isDirectory() && f.exists()) { - return; - } else if (!new File(dir).mkdir()) { - throw new GFacProviderException("Cannot make directory " + dir); - } - } - - public void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacProviderException { - ApplicationDeploymentDescriptionType app = jobExecutionContext. - getApplicationContext().getApplicationDeploymentDescription().getType(); - log.info("working diectroy = " + app.getStaticWorkingDirectory()); - log.info("temp directory = " + app.getScratchWorkingDirectory()); - makeFileSystemDir(app.getStaticWorkingDirectory()); - makeFileSystemDir(app.getScratchWorkingDirectory()); - makeFileSystemDir(app.getInputDataDirectory()); - makeFileSystemDir(app.getOutputDataDirectory()); - } - -} http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java b/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java index bade5f0..902e424 100644 --- a/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java +++ b/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java @@ -26,9 +26,9 @@ import org.apache.airavata.gfac.GFacException; import org.apache.airavata.gfac.context.ApplicationContext; import org.apache.airavata.gfac.context.JobExecutionContext; import org.apache.airavata.gfac.context.MessageContext; -import org.apache.airavata.gfac.handler.LocalDirectorySetupHandler; +import org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler; +import org.apache.airavata.gfac.local.impl.LocalProvider; import org.apache.airavata.gfac.provider.GFacProviderException; -import org.apache.airavata.gfac.provider.impl.LocalProvider; import org.apache.airavata.model.workspace.experiment.TaskDetails; import org.apache.airavata.persistance.registry.jpa.impl.LoggingRegistryImpl; import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/test/resources/gfac-config.xml ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/test/resources/gfac-config.xml b/modules/gfac/gfac-local/src/test/resources/gfac-config.xml index f16460f..b9432da 100644 --- a/modules/gfac/gfac-local/src/test/resources/gfac-config.xml +++ b/modules/gfac/gfac-local/src/test/resources/gfac-config.xml @@ -19,9 +19,9 @@ - + - + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java index ec14c08..6ab86af 100644 --- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java +++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java @@ -21,19 +21,13 @@ package org.apache.airavata.gfac.monitor.impl.pull.qstat; import org.apache.airavata.gfac.GFacException; -import org.apache.airavata.gfac.SecurityContext; -import org.apache.airavata.gfac.context.security.GSISecurityContext; +import org.apache.airavata.gfac.gsissh.security.GSISecurityContext; import org.apache.airavata.gfac.monitor.HostMonitorData; import org.apache.airavata.gfac.monitor.MonitorID; import org.apache.airavata.gsi.ssh.api.SSHApiException; -import org.apache.airavata.gsi.ssh.api.ServerInfo; -import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo; -import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration; import org.apache.airavata.gsi.ssh.impl.JobStatus; import org.apache.airavata.gsi.ssh.impl.PBSCluster; -import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.apache.airavata.model.workspace.experiment.JobState; -import org.apache.airavata.schemas.gfac.GsisshHostType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java deleted file mode 100644 index 2d71b7d..0000000 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.airavata.gfac.context.security; - -import java.io.IOException; - -import net.schmizz.sshj.SSHClient; -import net.schmizz.sshj.connection.channel.direct.Session; -import net.schmizz.sshj.userauth.keyprovider.KeyProvider; - -import org.apache.airavata.gfac.SecurityContext; -import org.apache.airavata.gsi.ssh.api.Cluster; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Handle SSH security - */ -public class SSHSecurityContext implements SecurityContext { - private static final Logger log = LoggerFactory.getLogger(SSHSecurityContext.class); - - public static final String SSH_SECURITY_CONTEXT = "ssh"; - - private String username; - private String privateKeyLoc; - private String keyPass; - private SSHClient sshClient; - private Session session; - - private Cluster pbsCluster; - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPrivateKeyLoc() { - return privateKeyLoc; - } - - public void setPrivateKeyLoc(String privateKeyLoc) { - this.privateKeyLoc = privateKeyLoc; - } - - public String getKeyPass() { - return keyPass; - } - - public void setKeyPass(String keyPass) { - this.keyPass = keyPass; - } - - public void closeSession(Session session) { - if (session != null) { - try { - session.close(); - } catch (Exception e) { - log.warn("Cannot Close SSH Session"); - } - } - } - - public Session getSession(String hostAddress) throws IOException { - try { - if (sshClient == null) { - sshClient = new SSHClient(); - } - if (getSSHClient().isConnected()) - return getSSHClient().startSession(); - - KeyProvider pkey = getSSHClient().loadKeys(getPrivateKeyLoc(), getKeyPass()); - - getSSHClient().loadKnownHosts(); - - getSSHClient().connect(hostAddress); - getSSHClient().authPublickey(getUsername(), pkey); - session = getSSHClient().startSession(); - return session; - - } catch (NullPointerException ne) { - throw new SecurityException("Cannot load security context for SSH", ne); - } - } - - public SSHClient getSSHClient() { - if (sshClient == null) { - sshClient = new SSHClient(); - } - return sshClient; - } - - public void setPbsCluster(Cluster pbsCluster) { - this.pbsCluster = pbsCluster; - } - - public Cluster getPbsCluster() { - return this.pbsCluster; - } -}