airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From scnakand...@apache.org
Subject [1/4] airavata git commit: copying gridftp client code from sandbox
Date Mon, 04 Jan 2016 19:35:05 GMT
Repository: airavata
Updated Branches:
  refs/heads/develop dceabc83e -> a7ff42e47


http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java
b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java
new file mode 100644
index 0000000..780c2b6
--- /dev/null
+++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/db/dao/FileTransferRequestDaoTest.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.file.manager.core.db.dao;
+
+import org.apache.airavata.model.file.transfer.FileTransferRequestModel;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+@Ignore
+public class FileTransferRequestDaoTest {
+    private final static Logger logger = LoggerFactory.getLogger(FileTransferRequestDaoTest.class);
+
+    @Test
+    public void testFileTransferRequestDao() throws IOException {
+        FileTransferRequestModel fileTransferRequestModel = new FileTransferRequestModel();
+        fileTransferRequestModel.setSrcHostCredToken("djkalbsbdaslfbalsfbslf");
+        fileTransferRequestModel.setSrcFilePath("test-file-path");
+        FileTransferRequestDao fileTransferRequestDao = new FileTransferRequestDao();
+        String transferId = fileTransferRequestDao.createFileTransferRequest(fileTransferRequestModel);
+        fileTransferRequestModel = fileTransferRequestDao.getFileTransferRequest(transferId);
+        System.out.println("Transfer Id:" + fileTransferRequestModel.getTransferId());
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java
b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java
new file mode 100644
index 0000000..68578db
--- /dev/null
+++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/CertFileReadTest.java
@@ -0,0 +1,126 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.file.manager.core.remote.client.gridftp;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.globus.gsi.SigningPolicy;
+import org.globus.gsi.SigningPolicyParser;
+import org.globus.gsi.util.CertificateIOUtil;
+import org.globus.util.GlobusResource;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import javax.security.auth.x500.X500Principal;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.security.MessageDigest;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Map;
+
+
+@Ignore("This test case used to debug JGlobus-102. No need to run this test with other gridftp
tests.")
+public class CertFileReadTest extends TestCase {
+
+    private static MessageDigest md5;
+
+    private static String CERT_FILE_LOCATION = "/Users/supun/Work/airavata-sandbox/grid-tools/certificates/";
+
+    @Test
+    public void testCertFileRead() throws Exception {
+
+        String path1 = CERT_FILE_LOCATION + "ffc3d59b";
+        String path2 = CERT_FILE_LOCATION + "e5cc84c2";
+
+
+        GlobusResource globusResource1 = new GlobusResource(path1 + ".signing_policy");
+        GlobusResource globusResource2 = new GlobusResource(path2 + ".signing_policy");
+
+        // ===== Testing globusResource1 - This should pass (cos no DC components) ================
//
+        X509Certificate crt1 = readCertificate(path1 + ".0");
+        X500Principal policySubjectCert1 = getPrincipal(globusResource1);
+
+        String certHash1 = CertificateIOUtil.nameHash(crt1.getSubjectX500Principal());
+        String principalHash1 = CertificateIOUtil.nameHash(policySubjectCert1);
+
+        System.out.println("======== Printing hashes for 1 ================");
+        System.out.println(certHash1);
+        System.out.println(principalHash1);
+
+        Assert.assertEquals("Certificate hash value does not match with the hash value generated
using principal name.",
+                certHash1, principalHash1);
+
+        // ===== Testing globusResource1 - This should fail (cos we have DC components) ================
//
+        X509Certificate crt2 = readCertificate(path2 + ".0");
+        X500Principal policySubjectCert2 = getPrincipal(globusResource2);
+
+        String certHash2 = CertificateIOUtil.nameHash(crt2.getSubjectX500Principal());
+        String principalHash2 = CertificateIOUtil.nameHash(policySubjectCert2);
+
+        System.out.println("======== Printing hashes for 2 ================");
+        System.out.println(certHash2);
+        System.out.println(principalHash2);
+
+        Assert.assertEquals("Certificate hash value does not match with the hash value generated
using principal name.",
+                certHash2, principalHash2);
+    }
+
+    private X500Principal getPrincipal(GlobusResource globusResource) throws Exception{
+
+        SigningPolicyParser parser = new SigningPolicyParser();
+
+        Reader reader = new InputStreamReader(globusResource.getInputStream());
+
+        Map<X500Principal, SigningPolicy> policies = parser.parse(reader);
+
+        return policies.keySet().iterator().next();
+
+    }
+
+    private X509Certificate readCertificate(String certPath) {
+        try {
+            FileInputStream fr = new FileInputStream(certPath);
+            CertificateFactory cf =
+                    CertificateFactory.getInstance("X509");
+            X509Certificate crt = (X509Certificate)
+                    cf.generateCertificate(fr);
+            System.out.println("Read certificate:");
+            System.out.println("\tCertificate for: " +
+                    crt.getSubjectDN());
+            System.out.println("\tCertificate issued by: " +
+                    crt.getIssuerDN());
+            System.out.println("\tCertificate is valid from " +
+                    crt.getNotBefore() + " to " + crt.getNotAfter());
+            System.out.println("\tCertificate SN# " +
+                    crt.getSerialNumber());
+            System.out.println("\tGenerated with " +
+                    crt.getSigAlgName());
+
+            return crt;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        return null;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java
b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java
new file mode 100644
index 0000000..13d427d
--- /dev/null
+++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/gridftp/FileTransferTest.java
@@ -0,0 +1,168 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.file.manager.core.remote.client.gridftp;
+
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.apache.airavata.file.manager.core.remote.client.gridftp.myproxy.SecurityContext;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.ietf.jgss.GSSCredential;
+import org.junit.Ignore;
+
+import java.io.*;
+import java.net.URI;
+
+
+@Ignore
+public class FileTransferTest extends TestCase {
+
+    private GSSCredential gssCredential;
+
+    private ExecutionContext executionContext;
+
+    private static final Logger log = Logger.getLogger(FileTransferTest.class);
+
+
+    public void setUp() throws Exception {
+
+        String userName = System.getProperty("myproxy.user");
+        String password = System.getProperty("myproxy.password");
+
+        SecurityContext context = null;
+
+        if (userName == null || password == null || userName.trim().equals("") || password.trim().equals(""))
{
+            log.error("myproxy.user and myproxy.password system properties are not set. Example
:- " +
+                    "> mvn clean install -Dmyproxy.user=u1 -Dmyproxy.password=xxx");
+
+            Assert.fail("Please set myproxy.user and myproxy.password system properties.");
+
+        } else {
+            context = new SecurityContext(userName, password);
+        }
+
+        log.info("Using my proxy user name - " + userName);
+
+        BasicConfigurator.configure();
+        Logger logger = Logger.getLogger("GridFTPClient");
+        Level lev = Level.toLevel("DEBUG");
+        logger.setLevel(lev);
+
+
+        context.login();
+        executionContext = new ExecutionContext();
+
+
+        String targeterp = executionContext.getGridFTPServerDestination();
+        String remoteDestFile = executionContext.getDestinationDataLocation();
+
+        URI dirLocation = GridFTP.createGsiftpURI(targeterp,
+                remoteDestFile.substring(0, remoteDestFile.lastIndexOf("/")));
+        gssCredential = context.getGssCredential();
+        System.out.println(dirLocation);
+
+    }
+
+    public void testMakeDir() throws Exception {
+
+        String targetErp = executionContext.getGridFTPServerDestination();
+        String remoteDestinationFile = executionContext.getDestinationDataLocation();
+
+        URI dirLocation = GridFTP.createGsiftpURI(targetErp,
+                remoteDestinationFile.substring(0, remoteDestinationFile.lastIndexOf("/")));
+
+        GridFTP ftp = new GridFTP();
+        ftp.makeDir(dirLocation, gssCredential);
+
+        Assert.assertTrue(ftp.exists(dirLocation, gssCredential));
+
+    }
+
+    public void testTransferData() throws Exception {
+
+        String sourceERP = executionContext.getGridFTPServerSource();
+        String remoteSrcFile = executionContext.getSourceDataLocation();
+
+        String targetErp = executionContext.getGridFTPServerDestination();
+        String remoteDestinationFile = executionContext.getDestinationDataLocation();
+
+        URI srcURI = GridFTP.createGsiftpURI(sourceERP, remoteSrcFile);
+        URI destURI = GridFTP.createGsiftpURI(targetErp, remoteDestinationFile);
+
+        GridFTP ftp = new GridFTP();
+        ftp.transfer(srcURI, destURI, gssCredential, true);
+
+        Assert.assertTrue(ftp.exists(destURI, gssCredential));
+
+    }
+
+    public void testDownloadFile() throws Exception {
+
+        String fileName = "./downloaded";
+
+        File deleteFile = new File(fileName);
+
+        if (deleteFile.exists()) {
+            if (!deleteFile.delete())
+                throw new RuntimeException("Unable to delete file " + fileName);
+        }
+
+        File f = new File(fileName);
+
+        GridFTP ftp = new GridFTP();
+        ftp.downloadFile(executionContext.getSourceDataFileUri(),
+                gssCredential, f);
+
+        Assert.assertTrue(f.exists());
+
+    }
+
+    public void testFileExists() throws Exception {
+
+        GridFTP ftp = new GridFTP();
+        Assert.assertTrue(ftp.exists(executionContext.getSourceDataFileUri(), gssCredential));
+    }
+
+    public void testUpdateFile() throws Exception {
+
+        String currentDir = System.getProperty("projectDirectory");
+
+        if (currentDir == null)
+            currentDir = "src/test/resources";
+        else
+            currentDir = currentDir + "/src/test/resources";
+
+        String file = currentDir + "/dummy";
+
+        System.out.println("File to upload is " + file);
+
+        File fileToUpload = new File(file);
+
+        Assert.assertTrue(fileToUpload.canRead());
+
+        GridFTP ftp = new GridFTP();
+        ftp.updateFile(executionContext.getUploadingFilePathUri(), gssCredential, fileToUpload);
+
+        Assert.assertTrue(ftp.exists(executionContext.getUploadingFilePathUri(), gssCredential));
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java
b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java
new file mode 100644
index 0000000..93fcba3
--- /dev/null
+++ b/modules/file-manager/file-manager-core/src/test/java/org/apache/airavata/file/manager/core/remote/client/scp/SCPStorageClientTest.java
@@ -0,0 +1,55 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.file.manager.core.remote.client.scp;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+@Ignore
+public class SCPStorageClientTest {
+    private final static Logger logger = LoggerFactory.getLogger(SCPStorageClientTest.class);
+
+    @Test
+    public void testSCPStorageClient() throws Exception {
+        File privateKey = new File("/Users/supun/.ssh/id_rsa");
+        byte[] privateKeyBytes = IOUtils.toByteArray(new FileInputStream(privateKey));
+
+        File publicKey = new File("/Users/supun/.ssh/id_rsa.pub");
+        byte[] publicKeyBytes = IOUtils.toByteArray(new FileInputStream(publicKey));
+
+        String passPhrase = "";
+        byte[] passPhraseBytes = passPhrase.getBytes();
+
+        SCPStorageClient scpStorageClient = new SCPStorageClient("gw75.iu.xsede.org", 22,
"pga", privateKeyBytes,
+                publicKeyBytes, passPhraseBytes);
+        File file = scpStorageClient.readFile("/var/www/portals/gateway-user-data/testdrive/test.txt");
+        System.out.println("File exists ? " + file.exists());
+        scpStorageClient.writeFile(file, "/var/www/portals/gateway-user-data/testdrive/test2.txt");
+        file.delete();
+        System.out.println("File exists ? " + file.exists());
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties
b/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties
new file mode 100644
index 0000000..407f09d
--- /dev/null
+++ b/modules/file-manager/file-manager-core/src/test/resources/airavata-myproxy-client.properties
@@ -0,0 +1,28 @@
+#Download trusted certificates from - https://software.xsede.org/security/xsede-certs.tar.gz
+trustedCertsFile=../certificates
+myproxyServer=myproxy.teragrid.org
+myproxy_lifetime=17280000
+myproxyPort=7512
+
+testing.host=trestles
+#testing.host=stampede
+#testing.host=lonestar
+
+## TACC Lonestar
+lonestar.gridftp.endpoint=gsiftp://gridftp1.ls4.tacc.utexas.edu:2811/
+
+## TACC Stampede
+stampede.gridftp.endpoint=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/
+
+## SDSC Trestles
+trestles.gridftp.endpoint=gsiftp://trestles.sdsc.edu:2811/
+
+gridftpserverSource=gsiftp://trestles-dm.sdsc.xsede.org:2811/
+gridftpSourcePath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/sample_wrfout.netcdf
+gridftpUploadingFilePath = /oasis/projects/nsf/sds128/ogce/file-transfer-tests/source/dummy1
+
+#gridftpserverDest=gsiftp://trestles-dm.sdsc.xsede.org:2811/
+#gridftpDestPath=/oasis/projects/nsf/sds128/ogce/file-transfer-tests/gf
+
+gridftpserverDest=gsiftp://gridftp.stampede.tacc.utexas.edu:2811/
+gridftpDestPath=/scratch/01437/ogce/file-transfer-tests/dest/xx
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/modules/file-manager/file-manager-core/src/test/resources/dummy
----------------------------------------------------------------------
diff --git a/modules/file-manager/file-manager-core/src/test/resources/dummy b/modules/file-manager/file-manager-core/src/test/resources/dummy
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/airavata/blob/9fd6be25/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift
----------------------------------------------------------------------
diff --git a/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift
b/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift
index ff9b932..4e2ea00 100644
--- a/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift
+++ b/thrift-interface-descriptions/data-models/file-manager-models/replica_models.thrift
@@ -64,7 +64,7 @@ struct FileModel {
     8: optional string sha256Checksum,
     9: optional FileModelType fileType,
    10: optional i32 fileSize,
-   11: optional string nativeFormat,
+   11: optional string dataType,
    12: optional i64 creationTime,
    13: optional i64 lastModifiedTime,
    14: optional list<FileReplicaModel> fileReplicas


Mime
View raw message