From commits-return-19622-apmail-airavata-commits-archive=airavata.apache.org@airavata.apache.org Wed Aug 29 19:45:15 2018 Return-Path: X-Original-To: apmail-airavata-commits-archive@www.apache.org Delivered-To: apmail-airavata-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 594B91ECA5 for ; Wed, 29 Aug 2018 19:45:15 +0000 (UTC) Received: (qmail 87902 invoked by uid 500); 29 Aug 2018 19:45:15 -0000 Delivered-To: apmail-airavata-commits-archive@airavata.apache.org Received: (qmail 87857 invoked by uid 500); 29 Aug 2018 19:45:15 -0000 Mailing-List: contact commits-help@airavata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airavata.apache.org Delivered-To: mailing list commits@airavata.apache.org Received: (qmail 87824 invoked by uid 99); 29 Aug 2018 19:45:15 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Aug 2018 19:45:15 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id A7C3885149; Wed, 29 Aug 2018 19:45:14 +0000 (UTC) Date: Wed, 29 Aug 2018 19:45:14 +0000 To: "commits@airavata.apache.org" Subject: [airavata] branch jupyter-integration updated: Optimizing directory listing in Adaptors MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <153557191460.24841.12147402588930763070@gitbox.apache.org> From: dimuthuupe@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: airavata X-Git-Refname: refs/heads/jupyter-integration X-Git-Reftype: branch X-Git-Oldrev: 95c4cf53c9aaea0adfab76e07ddb16aa85c3d341 X-Git-Newrev: 96a1529f73c86f171ba9f79dd38272cf0c205ab5 X-Git-Rev: 96a1529f73c86f171ba9f79dd38272cf0c205ab5 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch jupyter-integration in repository https://gitbox.apache.org/repos/asf/airavata.git The following commit(s) were added to refs/heads/jupyter-integration by this push: new 96a1529 Optimizing directory listing in Adaptors 96a1529 is described below commit 96a1529f73c86f171ba9f79dd38272cf0c205ab5 Author: dimuthu AuthorDate: Wed Aug 29 15:45:06 2018 -0400 Optimizing directory listing in Adaptors --- .../apache/airavata/agents/api/AgentAdaptor.java | 2 ++ .../org/apache/airavata/agents/api/FileInfo.java | 18 +++++++++++++++ .../agents/api/StorageResourceAdaptor.java | 1 + .../airavata/helix/agent/ssh/SshAgentAdaptor.java | 5 ++++ .../airavata/helix/adaptor/SSHJAgentAdaptor.java | 27 ++++++++++++++++++---- .../api/service/handler/RegistryServerHandler.java | 14 +++++++---- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java index 5355d5c..5e119f2 100644 --- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java +++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/AgentAdaptor.java @@ -46,6 +46,8 @@ public interface AgentAdaptor { public List listDirectory(String path) throws AgentException; + public List listDirectoryWithInfo(String path) throws AgentException; + public Boolean doesFileExist(String filePath) throws AgentException; public List getFileNameFromExtension(String fileName, String parentPath) throws AgentException; diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java index b435452..281a6cc 100644 --- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java +++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/FileInfo.java @@ -20,6 +20,8 @@ package org.apache.airavata.agents.api; public class FileInfo { + private String name; + private String path; private boolean isExist; private boolean isFile; private long size; @@ -27,6 +29,22 @@ public class FileInfo { private long modifiedDate; private String owner; + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + public boolean isExist() { return isExist; } diff --git a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java index d4c32f8..3bc9704 100644 --- a/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java +++ b/modules/airavata-helix/agent-api/src/main/java/org/apache/airavata/agents/api/StorageResourceAdaptor.java @@ -26,6 +26,7 @@ public interface StorageResourceAdaptor { public void uploadFile(String sourceFile, String destFile) throws AgentException; public void downloadFile(String sourceFile, String destFile) throws AgentException; public List listDirectory(String path) throws AgentException; + public List listDirectoryWithInfo(String path) throws AgentException; public void deleteFile(String path) throws AgentException; public void deleteDirectory(String path) throws AgentException; public Boolean doesFileExist(String filePath) throws AgentException; diff --git a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java index e4cffe9..9018f01 100644 --- a/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java +++ b/modules/airavata-helix/agent-impl/ssh-agent/src/main/java/org/apache/airavata/helix/agent/ssh/SshAgentAdaptor.java @@ -455,6 +455,11 @@ public class SshAgentAdaptor implements AgentAdaptor { } @Override + public List listDirectoryWithInfo(String path) throws AgentException { + throw new AgentException("Operation not implemented"); + } + + @Override public Boolean doesFileExist(String filePath) throws AgentException { String command = "ls " + filePath; ChannelExec channelExec = null; diff --git a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java index 182942b..54dc928 100644 --- a/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java +++ b/modules/airavata-helix/agent-impl/sshj-agent/src/main/java/org/apache/airavata/helix/adaptor/SSHJAgentAdaptor.java @@ -31,10 +31,7 @@ import net.schmizz.sshj.userauth.method.ChallengeResponseProvider; import net.schmizz.sshj.userauth.password.PasswordFinder; import net.schmizz.sshj.userauth.password.PasswordUtils; import net.schmizz.sshj.userauth.password.Resource; -import org.apache.airavata.agents.api.AgentAdaptor; -import org.apache.airavata.agents.api.AgentException; -import org.apache.airavata.agents.api.AgentUtils; -import org.apache.airavata.agents.api.CommandOutput; +import org.apache.airavata.agents.api.*; import org.apache.airavata.helix.adaptor.wrapper.SCPFileTransferWrapper; import org.apache.airavata.helix.agent.ssh.StandardOutReader; import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription; @@ -208,6 +205,28 @@ public class SSHJAgentAdaptor implements AgentAdaptor { } @Override + public List listDirectoryWithInfo(String path) throws AgentException { + try (SFTPClient sftpClient = sshjClient.newSFTPClientWrapper()) { + List ls = sftpClient.ls(path); + + return ls.stream().map(rInfo -> { + FileInfo fileInfo = new FileInfo(); + fileInfo.setName(rInfo.getName()); + fileInfo.setPath(rInfo.getPath()); + fileInfo.setExist(true); + fileInfo.setCreatedDate(rInfo.getAttributes().getAtime()); + fileInfo.setModifiedDate(rInfo.getAttributes().getMtime()); + fileInfo.setFile(rInfo.getAttributes().getType() != FileMode.Type.DIRECTORY); + fileInfo.setSize(rInfo.getAttributes().getSize()); + return fileInfo; + }).collect(Collectors.toList()); + + } catch (Exception e) { + throw new AgentException(e); + } + } + + @Override public Boolean doesFileExist(String filePath) throws AgentException { try (SFTPClient sftpClient = sshjClient.newSFTPClientWrapper()) { return sftpClient.statExistence(filePath) != null; diff --git a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java index 65ccb6f..359de71 100644 --- a/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java +++ b/modules/registry/registry-server/registry-api-service/src/main/java/org/apache/airavata/registry/api/service/handler/RegistryServerHandler.java @@ -5044,10 +5044,16 @@ public class RegistryServerHandler implements RegistryService.Iface { try { List fileStructures = new ArrayList<>(); - List fileNames = storageResourceAdaptor.listDirectory(dirPath); - for (String fileName : fileNames) { - String filePath = dirPath.endsWith(File.separator) ? dirPath + fileName : dirPath + File.separator + fileName; - fileStructures.add(getFileDetailsFromStorage(gatewayId, storageResourceId, userId, filePath)); + List fileInfos = storageResourceAdaptor.listDirectoryWithInfo(dirPath); + for (FileInfo fileInfo : fileInfos) { + + FileStructure fileStructure = new FileStructure(); + fileStructure.setName(fileInfo.getName()); + fileStructure.setPath(fileInfo.getPath()); + fileStructure.setCreatedDate(fileInfo.getCreatedDate()); + fileStructure.setModifiedDate(fileInfo.getModifiedDate()); + fileStructure.setIsFile(fileInfo.isFile()); + fileStructures.add(fileStructure); } return fileStructures; } catch (AgentException e) {