airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From machris...@apache.org
Subject [airavata-django-portal] 02/02: WIP
Date Wed, 15 May 2019 02:13:02 GMT
This is an automated email from the ASF dual-hosted git repository.

machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git

commit b71f79567d7e0645f2d873c85949ea0e492a0413
Author: Marcus Christie <machristie@apache.org>
AuthorDate: Tue May 14 22:12:31 2019 -0400

    WIP
---
 django_airavata/apps/api/data_products_helper.py |  69 +++++++++++----
 django_airavata/apps/api/datastore.py            | 106 ++++++-----------------
 django_airavata/apps/api/serializers.py          |   6 +-
 django_airavata/apps/api/views.py                |  36 ++------
 4 files changed, 86 insertions(+), 131 deletions(-)

diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index 65b8024..e301d78 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -1,4 +1,5 @@
 import os
+from urllib.parse import urlparse
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
@@ -15,21 +16,23 @@ from . import datastore, models
 
 
 def save(request, path, file):
-    # return data_product
-    # TODO
-    pass
+    "Save file in path in the user's storage."
+    username = request.user.username
+    full_path = datastore.save(username, path, file)
+    data_product = _save_data_product(request, full_path)
+    return data_product
 
 
 def open(request, data_product):
-    # return file object
-    # TODO
-    pass
+    "Return file object for replica if it exists in user storage."
+    path = _get_replica_filepath(data_product)
+    return datastore.open(request.user.username, path)
 
 
 def exists(request, data_product):
-    # return boolean
-    # TODO
-    pass
+    "Return True if replica for data_product exists in user storage."
+    path = _get_replica_filepath(data_product)
+    return datastore.exists(request.user.username, path)
 
 
 def delete(request, data_product):
@@ -58,6 +61,18 @@ def listdir(request, path):
         raise ObjectDoesNotExist("User storage path does not exist")
 
 
+def get_experiment_dir(request,
+                       project_name=None,
+                       experiment_name=None,
+                       path=None):
+    return datastore.get_experiment_dir(
+        request.user.username, project_name, experiment_name, path)
+
+
+def create_user_dir(request, path):
+    return datastore.create_user_dir(request.user.username, path)
+
+
 def _get_data_product_uri(request, full_path):
 
     user_file = models.User_Files.objects.filter(
@@ -65,17 +80,25 @@ def _get_data_product_uri(request, full_path):
     if user_file.exists():
         product_uri = user_file[0].file_dpu
     else:
-        data_product = _create_data_product(request.user.username, full_path)
-        product_uri = request.airavata_client.registerDataProduct(
-            request.authz_token, data_product)
-        user_file_instance = models.User_Files(
-            username=request.user.username,
-            file_path=full_path,
-            file_dpu=product_uri)
-        user_file_instance.save()
+        data_product = _save_data_product(request, full_path)
+        product_uri = data_product.productUri
     return product_uri
 
 
+def _save_data_product(request, full_path):
+    "Create, register and record in DB a data product for full_path."
+    data_product = _create_data_product(request.user.username, full_path)
+    product_uri = request.airavata_client.registerDataProduct(
+        request.authz_token, data_product)
+    data_product.productUri = product_uri
+    user_file_instance = models.User_Files(
+        username=request.user.username,
+        file_path=full_path,
+        file_dpu=product_uri)
+    user_file_instance.save()
+    return data_product
+
+
 def _create_data_product(username, full_path):
     data_product = DataProductModel()
     data_product.gatewayId = settings.GATEWAY_ID
@@ -97,3 +120,15 @@ def _create_data_product(username, full_path):
                               full_path)
     data_product.replicaLocations = [data_replica_location]
     return data_product
+
+
+def _get_replica_filepath(data_product):
+    replica_filepaths = [rep.filePath
+                         for rep in data_product.replicaLocations
+                         if rep.replicaLocationCategory ==
+                         ReplicaLocationCategory.GATEWAY_DATA_STORE]
+    replica_filepath = (replica_filepaths[0]
+                        if len(replica_filepaths) > 0 else None)
+    if replica_filepath:
+        return urlparse(replica_filepath).path
+    return None
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index 2b5f9b5..387bcb1 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -19,106 +19,49 @@ experiment_data_storage = FileSystemStorage(
 logger = logging.getLogger(__name__)
 
 
-# TODO: exists(username, path)
-def exists(data_product):
+def exists(username, path):
     """Check if replica for data product exists in this data store."""
-    filepath = _get_replica_filepath(data_product)
     try:
-        return experiment_data_storage.exists(filepath) if filepath else False
+        return _user_data_storage(username).exists(path)
     except SuspiciousFileOperation as e:
-        logger.warning("Unable to find file at {} for data product uri {}"
-                       .format(filepath, data_product.productUri))
+        logger.warning("Invalid path for user {}: {}".format(username, str(e)))
         return False
 
 
-# TODO: open(username, path)
-def open(data_product):
-    """Open replica for data product if it exists in this data store."""
-    if exists(data_product):
-        filepath = _get_replica_filepath(data_product)
-        return experiment_data_storage.open(filepath)
+def open(username, path):
+    """Open path for user if it exists in this data store."""
+    if exists(username, path):
+        return _user_data_storage(username).open(path)
     else:
-        raise ObjectDoesNotExist("Replica file does not exist")
-
-
-def save_user(username, file):
-    """Save file to username/project name/experiment_name in data store."""
-    user_dir = os.path.join(
-        experiment_data_storage.get_valid_name(username),
-        'MyFiles')
-    # file.name may be full path, so get just the name of the file
-    file_name = os.path.basename(file.name)
-    file_path = os.path.join(
-        user_dir,
-        experiment_data_storage.get_valid_name(file_name))
-    input_file_name = experiment_data_storage.save(file_path, file)
-    input_file_fullpath = experiment_data_storage.path(input_file_name)
-    # Create DataProductModel instance with DataReplicaLocationModel
-    data_product = DataProductModel()
-    data_product.gatewayId = settings.GATEWAY_ID
-    data_product.ownerName = username
-    data_product.productName = file_name
-    data_product.dataProductType = DataProductType.FILE
-    data_replica_location = DataReplicaLocationModel()
-    data_replica_location.storageResourceId = \
-        settings.GATEWAY_DATA_STORE_RESOURCE_ID
-    data_replica_location.replicaName = \
-        "{} gateway data store copy".format(file_name)
-    data_replica_location.replicaLocationCategory = \
-        ReplicaLocationCategory.GATEWAY_DATA_STORE
-    data_replica_location.replicaPersistentType = \
-        ReplicaPersistentType.TRANSIENT
-    data_replica_location.filePath = \
-        "file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
-                              input_file_fullpath)
-    data_product.replicaLocations = [data_replica_location]
-    return data_product
+        raise ObjectDoesNotExist("File path does not exist: {}".format(path))
 
 
-# TODO: save(username, path, file)
-def save(username, project_name, experiment_name, file):
+def save(username, path, file):
     """Save file to username/project name/experiment_name in data store."""
-    exp_dir = os.path.join(
-        _user_dir_name(username),
-        experiment_data_storage.get_valid_name(project_name),
-        experiment_data_storage.get_valid_name(experiment_name))
     # file.name may be full path, so get just the name of the file
     file_name = os.path.basename(file.name)
     file_path = os.path.join(
-        exp_dir,
-        experiment_data_storage.get_valid_name(file_name))
-    input_file_name = experiment_data_storage.save(file_path, file)
-    input_file_fullpath = experiment_data_storage.path(input_file_name)
-    # Create DataProductModel instance with DataReplicaLocationModel
-    data_product = _create_data_product(username, input_file_fullpath)
-    return data_product
-
-
-def save_user_file(username, path, file):
-    experiment_data_storage.save(os.path.join(
-        _user_dir_name(username),
-        experiment_data_storage.get_valid_name(path),
-        experiment_data_storage.get_valid_name(file.name)
-    ), file)
+        path, experiment_data_storage.get_valid_name(file_name))
+    user_data_storage = _user_data_storage(username)
+    input_file_name = user_data_storage.save(file_path, file)
+    input_file_fullpath = user_data_storage.path(input_file_name)
+    return input_file_fullpath
 
 
-def create_user_dir(username, path, dir_name):
-    user_dir = os.path.join(
-        _user_dir_name(username),
-        path,
-        experiment_data_storage.get_valid_name(dir_name))
-    if not experiment_data_storage.exists(user_dir):
-        os.mkdir(experiment_data_storage.path(user_dir))
+def create_user_dir(username, path):
+    user_data_storage = _user_data_storage(username)
+    user_dir = user_data_storage.get_valid_name(path)
+    if not user_data_storage.exists(user_dir):
+        os.mkdir(user_data_storage.path(user_dir))
     else:
         raise Exception(
-            "Directory {} already exists at that path".format(dir_name))
+            "Directory {} already exists".format(path))
 
 
-# TODO: copy(username, source_path, target_path)
-def copy(username, project_name, experiment_name, data_product):
-    """Copy a data product into username/project_name/experiment_name dir."""
-    f = open(data_product)
-    return save(username, project_name, experiment_name, f)
+def copy(username, source_path, target_path):
+    """Copy a user file into target_path dir."""
+    f = open(username, source_path)
+    return save(username, target_path, f)
 
 
 # TODO: delete(username, path)
@@ -138,6 +81,7 @@ def delete(data_product):
         raise ObjectDoesNotExist("Replica file does not exist")
 
 
+# TODO: update this to just return an available experiment directory name
 def get_experiment_dir(
         username,
         project_name=None,
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 86e235e..9c9fa08 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -52,7 +52,7 @@ from airavata.model.status.ttypes import ExperimentStatus
 from airavata.model.user.ttypes import UserProfile
 from airavata.model.workspace.ttypes import Project
 
-from . import datastore, models, thrift_utils
+from . import data_products_helper, datastore, models, thrift_utils
 
 log = logging.getLogger(__name__)
 
@@ -435,8 +435,8 @@ class DataProductSerializer(
 
     def get_downloadURL(self, data_product):
         """Getter for downloadURL field."""
-        if datastore.exists(data_product):
-            request = self.context['request']
+        request = self.context['request']
+        if data_products_helper.exists(request, data_product):
             return (request.build_absolute_uri(
                 reverse('django_airavata_api:download_file')) +
                 '?' +
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 21c7b6c..a7572a8 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -865,11 +865,11 @@ def upload_input_file(request):
             request.authz_token, project_id)
         exp_name = request.POST['experiment-name']
         input_file = request.FILES['file']
-        data_product = datastore.save(username, project.name, exp_name,
-                                      input_file)
-        data_product_uri = request.airavata_client.registerDataProduct(
-            request.authz_token, data_product)
-        data_product.productUri = data_product_uri
+        # TODO: experiment_data_dir should be passed in
+        experiment_data_dir = data_products_helper.get_experiment_dir(
+            request, project_name=project.name, experiment_name=exp_name)
+        data_product = data_products_helper.save(request, experiment_data_dir,
+                                                 input_file)
         serializer = serializers.DataProductSerializer(
             data_product, context={'request': request})
         return JsonResponse({'uploaded': True,
@@ -894,7 +894,7 @@ def download_file(request):
                     .format(data_product_uri), exc_info=True)
         raise Http404("data product does not exist") from e
     try:
-        data_file = datastore.open(data_product)
+        data_file = data_products_helper.open(request, data_product)
         response = FileResponse(data_file,
                                 content_type="application/octet-stream")
         file_name = os.path.basename(data_file.name)
@@ -1383,30 +1383,6 @@ class UserStoragePathView(APIView):
             context={'request': request})
         return Response(serializer.data)
 
-    def post(self, request, path="/", format=None):
-        # TODO: this needs to be fixed or rethought
-        username = request.user.username
-        user_storage_path = path
-        if user_storage_path.startswith("/"):
-            user_storage_path = "." + user_storage_path
-        serializer = self.serializer_class(
-            data=request.data, context={
-                'request': request})
-        serializer.is_valid(raise_exception=True)
-        if serializer.validated_data['type'] == 'file':
-            upload_file = request.FILES['file']
-            datastore.save_user_file(username, user_storage_path, upload_file)
-        elif serializer.validated_data['type'] == 'dir':
-            datastore.create_user_dir(
-                username, user_storage_path, serializer.validated_data['name'])
-
-        # TODO return representation of created item
-        listing = datastore.list_user_dir(
-            request.user.username, user_storage_path)
-        serializer = self.serializer_class(
-            listing, many=True, context={'request': request})
-        return Response(serializer.data)
-
 
 class WorkspacePreferencesView(APIView):
     serializer_class = serializers.WorkspacePreferencesSerializer


Mime
View raw message