airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From machris...@apache.org
Subject [airavata-django-portal] 01/02: AIRAVATA-3033 REST API for browsing, downloading user storage
Date Wed, 15 May 2019 02:13:01 GMT
This is an automated email from the ASF dual-hosted git repository.

machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git

commit 88089904f303826f97b188cab4cdb0cc9c6007f6
Author: Marcus Christie <machristie@apache.org>
AuthorDate: Tue May 14 09:40:14 2019 -0400

    AIRAVATA-3033 REST API for browsing, downloading user storage
---
 django_airavata/apps/api/data_products_helper.py   |  99 ++++++++++++++
 django_airavata/apps/api/datastore.py              |  36 +++++
 .../apps/api/migrations/0002_auto_20190513_2037.py |  27 ++++
 django_airavata/apps/api/models.py                 |  12 ++
 django_airavata/apps/api/serializers.py            |  26 ++++
 django_airavata/apps/api/urls.py                   |  11 +-
 django_airavata/apps/api/views.py                  | 150 ++++++++++++++-------
 .../workspace/migrations/0002_delete_user_files.py |  18 +++
 django_airavata/apps/workspace/models.py           |   8 --
 9 files changed, 329 insertions(+), 58 deletions(-)

diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
new file mode 100644
index 0000000..65b8024
--- /dev/null
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -0,0 +1,99 @@
+import os
+
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+
+from airavata.model.data.replica.ttypes import (
+    DataProductModel,
+    DataProductType,
+    DataReplicaLocationModel,
+    ReplicaLocationCategory,
+    ReplicaPersistentType
+)
+
+from . import datastore, models
+
+
+def save(request, path, file):
+    # return data_product
+    # TODO
+    pass
+
+
+def open(request, data_product):
+    # return file object
+    # TODO
+    pass
+
+
+def exists(request, data_product):
+    # return boolean
+    # TODO
+    pass
+
+
+def delete(request, data_product):
+    # TODO
+    pass
+
+
+def listdir(request, path):
+    if datastore.user_file_exists(request.user.username, path):
+        directories, files = datastore.list_user_dir(
+            request.user.username, path)
+        directories_data = []
+        for d in directories:
+            directories_data.append({'name': d,
+                                     'path': os.path.join(path, d)})
+        files_data = []
+        for f in files:
+            user_rel_path = os.path.join(path, f)
+            full_path = datastore.path(request.user.username, user_rel_path)
+            data_product_uri = _get_data_product_uri(request, full_path)
+            files_data.append({'name': f,
+                               'path': user_rel_path,
+                               'data-product-uri': data_product_uri})
+        return directories_data, files_data
+    else:
+        raise ObjectDoesNotExist("User storage path does not exist")
+
+
+def _get_data_product_uri(request, full_path):
+
+    user_file = models.User_Files.objects.filter(
+        username=request.user.username, file_path=full_path)
+    if user_file.exists():
+        product_uri = user_file[0].file_dpu
+    else:
+        data_product = _create_data_product(request.user.username, full_path)
+        product_uri = request.airavata_client.registerDataProduct(
+            request.authz_token, data_product)
+        user_file_instance = models.User_Files(
+            username=request.user.username,
+            file_path=full_path,
+            file_dpu=product_uri)
+        user_file_instance.save()
+    return product_uri
+
+
+def _create_data_product(username, full_path):
+    data_product = DataProductModel()
+    data_product.gatewayId = settings.GATEWAY_ID
+    data_product.ownerName = username
+    file_name = os.path.basename(full_path)
+    data_product.productName = file_name
+    data_product.dataProductType = DataProductType.FILE
+    data_replica_location = DataReplicaLocationModel()
+    data_replica_location.storageResourceId = \
+        settings.GATEWAY_DATA_STORE_RESOURCE_ID
+    data_replica_location.replicaName = \
+        "{} gateway data store copy".format(file_name)
+    data_replica_location.replicaLocationCategory = \
+        ReplicaLocationCategory.GATEWAY_DATA_STORE
+    data_replica_location.replicaPersistentType = \
+        ReplicaPersistentType.TRANSIENT
+    data_replica_location.filePath = \
+        "file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
+                              full_path)
+    data_product.replicaLocations = [data_replica_location]
+    return data_product
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index eb0de66..2b5f9b5 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -19,6 +19,7 @@ experiment_data_storage = FileSystemStorage(
 logger = logging.getLogger(__name__)
 
 
+# TODO: exists(username, path)
 def exists(data_product):
     """Check if replica for data product exists in this data store."""
     filepath = _get_replica_filepath(data_product)
@@ -30,6 +31,7 @@ def exists(data_product):
         return False
 
 
+# TODO: open(username, path)
 def open(data_product):
     """Open replica for data product if it exists in this data store."""
     if exists(data_product):
@@ -73,6 +75,7 @@ def save_user(username, file):
     return data_product
 
 
+# TODO: save(username, path, file)
 def save(username, project_name, experiment_name, file):
     """Save file to username/project name/experiment_name in data store."""
     exp_dir = os.path.join(
@@ -91,12 +94,34 @@ def save(username, project_name, experiment_name, file):
     return data_product
 
 
+def save_user_file(username, path, file):
+    experiment_data_storage.save(os.path.join(
+        _user_dir_name(username),
+        experiment_data_storage.get_valid_name(path),
+        experiment_data_storage.get_valid_name(file.name)
+    ), file)
+
+
+def create_user_dir(username, path, dir_name):
+    user_dir = os.path.join(
+        _user_dir_name(username),
+        path,
+        experiment_data_storage.get_valid_name(dir_name))
+    if not experiment_data_storage.exists(user_dir):
+        os.mkdir(experiment_data_storage.path(user_dir))
+    else:
+        raise Exception(
+            "Directory {} already exists at that path".format(dir_name))
+
+
+# TODO: copy(username, source_path, target_path)
 def copy(username, project_name, experiment_name, data_product):
     """Copy a data product into username/project_name/experiment_name dir."""
     f = open(data_product)
     return save(username, project_name, experiment_name, f)
 
 
+# TODO: delete(username, path)
 def delete(data_product):
 
     """Delete replica for data product in this data store."""
@@ -162,6 +187,17 @@ def get_data_product(username, file_path):
         raise ObjectDoesNotExist("User file does not exist")
 
 
+def list_user_dir(username, file_path):
+    logger.debug("file_path={}".format(file_path))
+    user_data_storage = _user_data_storage(username)
+    return user_data_storage.listdir(file_path)
+
+
+def path(username, file_path):
+    user_data_storage = _user_data_storage(username)
+    return user_data_storage.path(file_path)
+
+
 def _get_replica_filepath(data_product):
     replica_filepaths = [rep.filePath
                          for rep in data_product.replicaLocations
diff --git a/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py b/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py
new file mode 100644
index 0000000..4bd4988
--- /dev/null
+++ b/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-13 20:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_airavata_api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='User_Files',
+            fields=[
+                ('username', models.CharField(max_length=64)),
+                ('file_path', models.TextField()),
+                ('file_dpu', models.CharField(max_length=500, primary_key=True, serialize=False)),
+            ],
+        ),
+        migrations.AddIndex(
+            model_name='user_files',
+            index=models.Index(fields=['username', 'file_path'], name='username_file_path_idx'),
+        ),
+    ]
diff --git a/django_airavata/apps/api/models.py b/django_airavata/apps/api/models.py
index 92344fd..5242208 100644
--- a/django_airavata/apps/api/models.py
+++ b/django_airavata/apps/api/models.py
@@ -9,3 +9,15 @@ class WorkspacePreferences(models.Model):
     @classmethod
     def create(self, username):
         return WorkspacePreferences(username=username)
+
+
+class User_Files(models.Model):
+    username = models.CharField(max_length=64)
+    file_path = models.TextField()
+    file_dpu = models.CharField(max_length=500, primary_key=True)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=['username', 'file_path'],
+                         name='username_file_path_idx')
+        ]
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 5f0685c..86e235e 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -748,6 +748,32 @@ class ParserSerializer(thrift_utils.create_serializer_class(Parser)):
         lookup_url_kwarg='parser_id')
 
 
+class UserStorageFileSerializer(serializers.Serializer):
+    name = serializers.CharField()
+    downloadURL = serializers.SerializerMethodField()
+
+    def get_downloadURL(self, file):
+        """Getter for downloadURL field."""
+        request = self.context['request']
+        return (request.build_absolute_uri(
+            reverse('django_airavata_api:download_file')) +
+            '?' +
+            urlencode({'data-product-uri': file['data-product-uri']}))
+
+
+class UserStorageDirectorySerializer(serializers.Serializer):
+    name = serializers.CharField()
+    url = FullyEncodedHyperlinkedIdentityField(
+        view_name='django_airavata_api:user-storage-items',
+        lookup_field='path',
+        lookup_url_kwarg='path')
+
+
+class UserStoragePathSerializer(serializers.Serializer):
+    directories = UserStorageDirectorySerializer(many=True)
+    files = UserStorageFileSerializer(many=True)
+
+
 # ModelSerializers
 class WorkspacePreferencesSerializer(serializers.ModelSerializer):
     class Meta:
diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py
index d835967..6b80b19 100644
--- a/django_airavata/apps/api/urls.py
+++ b/django_airavata/apps/api/urls.py
@@ -45,9 +45,10 @@ router.register(r'parsers', views.ParserViewSet, base_name='parser')
 app_name = 'django_airavata_api'
 urlpatterns = [
     url(r'^', include(router.urls)),
-    url(r'^get-ufiles$', views.get_user_files, name='get_user_files'),
-    url(r'^upload-ufiles$', views.upload_user_file, name='upload_user_file'),
-    url(r'^delete-ufiles$', views.delete_user_file, name='delete_user_file'),
+    # TODO: remove these
+    # url(r'^get-ufiles$', views.get_user_files, name='get_user_files'),
+    # url(r'^upload-ufiles$', views.upload_user_file, name='upload_user_file'),
+    # url(r'^delete-ufiles$', views.delete_user_file, name='delete_user_file'),
     url(r'^upload$', views.upload_input_file, name='upload_input_file'),
     url(r'^download', views.download_file, name='download_file'),
     url(r'^delete-file$', views.delete_file, name='delete_file'),
@@ -77,6 +78,10 @@ urlpatterns = [
     url(r'^workspace-preferences',
         views.WorkspacePreferencesView.as_view(),
         name="workspace-preferences"),
+    # url(r'^user-storage/~/(?P<path>.*)/$',
+    url(r'^user-storage/~/(?P<path>.*)$',
+        views.UserStoragePathView.as_view(),
+        name="user-storage-items")
 ]
 
 if logger.isEnabledFor(logging.DEBUG):
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 989e4ae..21c7b6c 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -37,9 +37,18 @@ from django_airavata.apps.api.view_utils import (
     APIResultPagination,
     GenericAPIBackedViewSet
 )
-from django_airavata.apps.workspace.models import User_Files
 
-from . import datastore, helpers, models, serializers, thrift_utils
+from . import (
+    data_products_helper,
+    datastore,
+    helpers,
+    models,
+    serializers,
+    thrift_utils
+)
+
+# from django_airavata.apps.workspace.models import User_Files
+
 
 READ_PERMISSION_TYPE = '{}:READ'
 
@@ -788,63 +797,63 @@ class DataProductView(APIView):
             data_product, context={'request': request})
         return Response(serializer.data)
 
+# TODO: remove
+# @login_required
+# def get_user_files(request):
 
-@login_required
-def get_user_files(request):
+#         dirs = []      # a list with file_name and file_dpu for each file
+#         for o in User_Files.objects.values_list('file_name', 'file_dpu'):
+#             file_details = {}
+#             file_details['file_name'] = o[0]
+#             file_details['file_dpu'] = o[1]
+#             dirs.append(file_details)
 
-        dirs = []      # a list with file_name and file_dpu for each file
-        for o in User_Files.objects.values_list('file_name', 'file_dpu'):
-            file_details = {}
-            file_details['file_name'] = o[0]
-            file_details['file_dpu'] = o[1]
-            dirs.append(file_details)
+#         return JsonResponse({'uploaded': True, 'user-files': dirs})
 
-        return JsonResponse({'uploaded': True, 'user-files': dirs})
 
+# @login_required
+# def upload_user_file(request):
+#         username = request.user.username
+#         input_file = request.FILES['file']
+#         file_details = {}
 
-@login_required
-def upload_user_file(request):
-        username = request.user.username
-        input_file = request.FILES['file']
-        file_details = {}
-
-        # To avoid duplicate file names
+#         # To avoid duplicate file names
 
-        if User_Files.objects.filter(file_name=input_file.name).exists():
-            resp = JsonResponse({'uploaded': False, 'error': "File already exists"})
-            resp.status_code = 400
-            return resp
+#         if User_Files.objects.filter(file_name=input_file.name).exists():
+#             resp = JsonResponse({'uploaded': False, 'error': "File already exists"})
+#             resp.status_code = 400
+#             return resp
 
-        else:
+#         else:
 
-            data_product = datastore.save_user(username, input_file)
-            data_product_uri = request.airavata_client.registerDataProduct(
-                request.authz_token, data_product)
-            d = User_Files(file_name=input_file.name, file_dpu=data_product_uri)
-            d.save()
-            file_details['file_name'] = d.file_name
-            file_details['file_dpu'] = d.file_dpu
-            return JsonResponse({'uploaded': True,
-                                 'upload-file': file_details})
+#             data_product = datastore.save_user(username, input_file)
+#             data_product_uri = request.airavata_client.registerDataProduct(
+#                 request.authz_token, data_product)
+#             d = User_Files(file_name=input_file.name, file_dpu=data_product_uri)
+#             d.save()
+#             file_details['file_name'] = d.file_name
+#             file_details['file_dpu'] = d.file_dpu
+#             return JsonResponse({'uploaded': True,
+#                                  'upload-file': file_details})
 
 
-@login_required
-def delete_user_file(request):
-        data_product_uri = request.body.decode('utf-8')
-        try:
-            data_product = request.airavata_client.getDataProduct(
-                request.authz_token, data_product_uri)
+# @login_required
+# def delete_user_file(request):
+#         data_product_uri = request.body.decode('utf-8')
+#         try:
+#             data_product = request.airavata_client.getDataProduct(
+#                 request.authz_token, data_product_uri)
 
-        except Exception as e:
-            log.warning("Failed to load DataProduct for {}"
-                        .format(data_product_uri), exc_info=True)
-            raise Http404("data product does not exist")(e)
+#         except Exception as e:
+#             log.warning("Failed to load DataProduct for {}"
+#                         .format(data_product_uri), exc_info=True)
+#             raise Http404("data product does not exist")(e)
 
-        # remove file_details entry from database and delete from datastore
-        User_Files.objects.filter(file_dpu=data_product_uri).delete()
-        datastore.delete(data_product)
+#         # remove file_details entry from database and delete from datastore
+#         User_Files.objects.filter(file_dpu=data_product_uri).delete()
+#         datastore.delete(data_product)
 
-        return JsonResponse({'deleted': True})
+#         return JsonResponse({'deleted': True})
 
 
 @login_required
@@ -897,6 +906,13 @@ def download_file(request):
 
 
 @login_required
+def user_storage_download_file(request, path):
+    user_storage_path = path
+    if user_storage_path.startswith("/"):
+        user_storage_path = "." + user_storage_path
+
+
+@login_required
 def delete_file(request):
     # TODO check that user has write access to this file using sharing API
     data_product_uri = request.GET.get('data-product-uri', '')
@@ -1337,7 +1353,8 @@ class ParserViewSet(mixins.CreateModelMixin,
     lookup_field = 'parser_id'
 
     def get_list(self):
-        return self.request.airavata_client.listAllParsers(self.authz_token, settings.GATEWAY_ID)
+        return self.request.airavata_client.listAllParsers(
+            self.authz_token, settings.GATEWAY_ID)
 
     def get_instance(self, lookup_value):
         return self.request.airavata_client.getParser(
@@ -1352,6 +1369,45 @@ class ParserViewSet(mixins.CreateModelMixin,
         self.request.airavata_client.saveParser(self.authz_token, parser)
 
 
+class UserStoragePathView(APIView):
+    serializer_class = serializers.UserStoragePathSerializer
+
+    def get(self, request, path="/", format=None):
+        user_storage_path = path
+        if user_storage_path.startswith("/"):
+            user_storage_path = "." + user_storage_path
+        # TODO: check if path is directory or file
+        directories, files = data_products_helper.listdir(request, path)
+        serializer = self.serializer_class(
+            {'directories': directories, 'files': files},
+            context={'request': request})
+        return Response(serializer.data)
+
+    def post(self, request, path="/", format=None):
+        # TODO: this needs to be fixed or rethought
+        username = request.user.username
+        user_storage_path = path
+        if user_storage_path.startswith("/"):
+            user_storage_path = "." + user_storage_path
+        serializer = self.serializer_class(
+            data=request.data, context={
+                'request': request})
+        serializer.is_valid(raise_exception=True)
+        if serializer.validated_data['type'] == 'file':
+            upload_file = request.FILES['file']
+            datastore.save_user_file(username, user_storage_path, upload_file)
+        elif serializer.validated_data['type'] == 'dir':
+            datastore.create_user_dir(
+                username, user_storage_path, serializer.validated_data['name'])
+
+        # TODO return representation of created item
+        listing = datastore.list_user_dir(
+            request.user.username, user_storage_path)
+        serializer = self.serializer_class(
+            listing, many=True, context={'request': request})
+        return Response(serializer.data)
+
+
 class WorkspacePreferencesView(APIView):
     serializer_class = serializers.WorkspacePreferencesSerializer
 
diff --git a/django_airavata/apps/workspace/migrations/0002_delete_user_files.py b/django_airavata/apps/workspace/migrations/0002_delete_user_files.py
new file mode 100644
index 0000000..9ed44e7
--- /dev/null
+++ b/django_airavata/apps/workspace/migrations/0002_delete_user_files.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-10 16:35
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_airavata_workspace', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='User_Files',
+        ),
+    ]
diff --git a/django_airavata/apps/workspace/models.py b/django_airavata/apps/workspace/models.py
index a17940b..35e0d64 100644
--- a/django_airavata/apps/workspace/models.py
+++ b/django_airavata/apps/workspace/models.py
@@ -1,10 +1,2 @@
 
 # Create your models here.
-
-from django.db import models
-
-
-class User_Files(models.Model):
-    file_name = models.CharField(max_length=50)
-    file_dpu = models.CharField(max_length=500)
-    # username = models.CharField(primary_key=True,max_length=20) not required


Mime
View raw message