airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dimuthu...@apache.org
Subject [airavata] branch jupyter-integration updated: Optimizing directory listing in Adaptors
Date Wed, 29 Aug 2018 19:45:14 GMT
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch jupyter-integration
in repository https://gitbox.apache.org/repos/asf/airavata.git


The following commit(s) were added to refs/heads/jupyter-integration by this push:
     new 96a1529  Optimizing directory listing in Adaptors
96a1529 is described below

commit 96a1529f73c86f171ba9f79dd38272cf0c205ab5
Author: dimuthu <dimuthu.upeksha2@gmail.com>
AuthorDate: Wed Aug 29 15:45:06 2018 -0400

    Optimizing directory listing in Adaptors
---
 .../apache/airavata/agents/api/AgentAdaptor.java   |  2 ++
 .../org/apache/airavata/agents/api/FileInfo.java   | 18 +++++++++++++++
 .../agents/api/StorageResourceAdaptor.java         |  1 +
 .../airavata/helix/agent/ssh/SshAgentAdaptor.java  |  5 ++++
 .../airavata/helix/adaptor/SSHJAgentAdaptor.java   | 27 ++++++++++++++++++----
 .../api/service/handler/RegistryServerHandler.java | 14 +++++++----
 6 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
index 5355d5c..5e119f2 100644
--- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
+++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java
@@ -46,6 +46,8 @@ public interface AgentAdaptor {
 
     public List<String> listDirectory(String path) throws AgentException;
 
+    public List<FileInfo> listDirectoryWithInfo(String path) throws AgentException;
+
     public Boolean doesFileExist(String filePath) throws AgentException;
 
     public List<String> getFileNameFromExtension(String fileName, String parentPath)
throws AgentException;
diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
index b435452..281a6cc 100644
--- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
+++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java
@@ -20,6 +20,8 @@
 package org.apache.airavata.agents.api;
 
 public class FileInfo {
+    private String name;
+    private String path;
     private boolean isExist;
     private boolean isFile;
     private long size;
@@ -27,6 +29,22 @@ public class FileInfo {
     private long modifiedDate;
     private String owner;
 
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+
     public boolean isExist() {
         return isExist;
     }
diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
index d4c32f8..3bc9704 100644
--- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
+++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java
@@ -26,6 +26,7 @@ public interface StorageResourceAdaptor {
     public void uploadFile(String sourceFile, String destFile) throws AgentException;
     public void downloadFile(String sourceFile, String destFile) throws AgentException;
     public List<String> listDirectory(String path) throws AgentException;
+    public List<FileInfo> listDirectoryWithInfo(String path) throws AgentException;
     public void deleteFile(String path) throws AgentException;
     public void deleteDirectory(String path) throws AgentException;
     public Boolean doesFileExist(String filePath) throws AgentException;
diff --git a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
index e4cffe9..9018f01 100644
--- a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
+++ b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java
@@ -455,6 +455,11 @@ public class SshAgentAdaptor implements AgentAdaptor {
     }
 
     @Override
+    public List<FileInfo> listDirectoryWithInfo(String path) throws AgentException
{
+        throw new AgentException("Operation not implemented");
+    }
+
+    @Override
     public Boolean doesFileExist(String filePath) throws AgentException {
         String command = "ls " + filePath;
         ChannelExec channelExec = null;
diff --git a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
index 182942b..54dc928 100644
--- a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
+++ b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java
@@ -31,10 +31,7 @@ import net.schmizz.sshj.userauth.method.ChallengeResponseProvider;
 import net.schmizz.sshj.userauth.password.PasswordFinder;
 import net.schmizz.sshj.userauth.password.PasswordUtils;
 import net.schmizz.sshj.userauth.password.Resource;
-import org.apache.airavata.agents.api.AgentAdaptor;
-import org.apache.airavata.agents.api.AgentException;
-import org.apache.airavata.agents.api.AgentUtils;
-import org.apache.airavata.agents.api.CommandOutput;
+import org.apache.airavata.agents.api.*;
 import org.apache.airavata.helix.adaptor.wrapper.SCPFileTransferWrapper;
 import org.apache.airavata.helix.agent.ssh.StandardOutReader;
 import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
@@ -208,6 +205,28 @@ public class SSHJAgentAdaptor implements AgentAdaptor {
     }
 
     @Override
+    public List<FileInfo> listDirectoryWithInfo(String path) throws AgentException
{
+        try (SFTPClient sftpClient = sshjClient.newSFTPClientWrapper()) {
+            List<RemoteResourceInfo> ls = sftpClient.ls(path);
+
+            return ls.stream().map(rInfo -> {
+                FileInfo fileInfo = new FileInfo();
+                fileInfo.setName(rInfo.getName());
+                fileInfo.setPath(rInfo.getPath());
+                fileInfo.setExist(true);
+                fileInfo.setCreatedDate(rInfo.getAttributes().getAtime());
+                fileInfo.setModifiedDate(rInfo.getAttributes().getMtime());
+                fileInfo.setFile(rInfo.getAttributes().getType() != FileMode.Type.DIRECTORY);
+                fileInfo.setSize(rInfo.getAttributes().getSize());
+                return fileInfo;
+            }).collect(Collectors.toList());
+
+        } catch (Exception e) {
+            throw new AgentException(e);
+        }
+    }
+
+    @Override
     public Boolean doesFileExist(String filePath) throws AgentException {
         try (SFTPClient sftpClient = sshjClient.newSFTPClientWrapper()) {
             return sftpClient.statExistence(filePath) != null;
diff --git a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
index 65ccb6f..359de71 100644
--- a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
+++ b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java
@@ -5044,10 +5044,16 @@ public class RegistryServerHandler implements RegistryService.Iface
{
 
         try {
             List<FileStructure> fileStructures = new ArrayList<>();
-            List<String> fileNames = storageResourceAdaptor.listDirectory(dirPath);
-            for (String fileName : fileNames) {
-                String filePath = dirPath.endsWith(File.separator) ? dirPath + fileName :
dirPath + File.separator + fileName;
-                fileStructures.add(getFileDetailsFromStorage(gatewayId, storageResourceId,
userId, filePath));
+            List<FileInfo> fileInfos = storageResourceAdaptor.listDirectoryWithInfo(dirPath);
+            for (FileInfo fileInfo : fileInfos) {
+
+                FileStructure fileStructure = new FileStructure();
+                fileStructure.setName(fileInfo.getName());
+                fileStructure.setPath(fileInfo.getPath());
+                fileStructure.setCreatedDate(fileInfo.getCreatedDate());
+                fileStructure.setModifiedDate(fileInfo.getModifiedDate());
+                fileStructure.setIsFile(fileInfo.isFile());
+                fileStructures.add(fileStructure);
             }
             return  fileStructures;
         } catch (AgentException e) {


Mime
View raw message