airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From machris...@apache.org
Subject [airavata-django-portal] 02/03: AIRAVATA-2616 Allow removing and changing input files
Date Thu, 13 Dec 2018 19:59:44 GMT
This is an automated email from the ASF dual-hosted git repository.

machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git

commit cfc33747f98593fb3b237935045bff618ee11273
Author: Marcus Christie <machrist@iu.edu>
AuthorDate: Thu Dec 13 14:49:08 2018 -0500

    AIRAVATA-2616 Allow removing and changing input files
---
 django_airavata/apps/api/datastore.py              | 14 ++++
 django_airavata/apps/api/serializers.py            | 24 ++++++-
 .../api/static/django_airavata_api/js/index.js     |  3 +
 .../django_airavata_api/js/service_config.js       | 10 +++
 django_airavata/apps/api/urls.py                   |  3 +
 django_airavata/apps/api/views.py                  | 41 +++++++++---
 .../js/components/experiment/DataProductViewer.vue | 40 ++++++++++++
 .../js/components/experiment/ExperimentSummary.vue | 31 ++++-----
 .../experiment/input-editors/FileInputEditor.vue   | 76 +++++++++++++++++++---
 .../js/containers/EditExperimentContainer.vue      |  2 +-
 10 files changed, 205 insertions(+), 39 deletions(-)

diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index aa4c17b..16576c0 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -72,6 +72,20 @@ def save(username, project_name, experiment_name, file):
     return data_product
 
 
+def delete(data_product):
+    """Delete replica for data product in this data store."""
+    if exists(data_product):
+        filepath = _get_replica_filepath(data_product)
+        try:
+            experiment_data_storage.delete(filepath)
+        except Exception as e:
+            logger.error("Unable to delete file {} for data product uri {}"
+                         .format(filepath, data_product.productUri))
+            raise
+    else:
+        raise ObjectDoesNotExist("Replica file does not exist")
+
+
 def get_experiment_dir(username, project_name, experiment_name):
     """Return an experiment directory (full path) for the given experiment."""
     experiment_dir_name = os.path.join(
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 13bbeb5..74cdf04 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -33,7 +33,7 @@ from airavata.model.appcatalog.parser.ttypes import Parser
 from airavata.model.appcatalog.storageresource.ttypes import (
     StorageResourceDescription
 )
-from airavata.model.application.io.ttypes import InputDataObjectType
+from airavata.model.application.io.ttypes import DataType, InputDataObjectType
 from airavata.model.credential.store.ttypes import (
     CredentialSummary,
     SummaryType
@@ -419,6 +419,24 @@ class ExperimentSerializer(
             request.authz_token, experiment.experimentId,
             ResourcePermissionType.WRITE)
 
+    def update(self, instance, validated_data):
+        result = super().update(instance, validated_data)
+        removed_input_files = self._find_removed_input_files(
+            instance.experimentInputs, result.experimentInputs)
+        result._removed_input_files = removed_input_files
+        return result
+
+    def _find_removed_input_files(self,
+                                  old_experiment_inputs,
+                                  new_experiment_inputs):
+        old_input_data_product_uris = set(
+            inp.value for inp in old_experiment_inputs
+            if inp.type == DataType.URI)
+        new_input_data_product_uris = set(
+            inp.value for inp in new_experiment_inputs
+            if inp.type == DataType.URI)
+        return old_input_data_product_uris - new_input_data_product_uris
+
 
 class DataReplicaLocationSerializer(
         thrift_utils.create_serializer_class(DataReplicaLocationModel)):
@@ -432,6 +450,10 @@ class DataProductSerializer(
     lastModifiedTime = UTCPosixTimestampDateTimeField()
     replicaLocations = DataReplicaLocationSerializer(many=True)
     downloadURL = serializers.SerializerMethodField()
+    url = FullyEncodedHyperlinkedIdentityField(
+        view_name='django_airavata_api:data-product-detail',
+        lookup_field='productUri',
+        lookup_url_kwarg='product_uri')
 
     def get_downloadURL(self, data_product):
         """Getter for downloadURL field."""
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js
index 34842a9..406494f 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/index.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js
@@ -11,6 +11,7 @@ import BatchQueue from "./models/BatchQueue";
 import BatchQueueResourcePolicy from "./models/BatchQueueResourcePolicy";
 import CommandObject from "./models/CommandObject";
 import ComputeResourcePolicy from "./models/ComputeResourcePolicy";
+import DataProduct from "./models/DataProduct";
 import DataType from "./models/DataType";
 import Experiment from "./models/Experiment";
 import ExperimentState from "./models/ExperimentState";
@@ -63,6 +64,7 @@ exports.models = {
   BatchQueueResourcePolicy,
   CommandObject,
   ComputeResourcePolicy,
+  DataProduct,
   DataType,
   Experiment,
   ExperimentState,
@@ -92,6 +94,7 @@ exports.services = {
   CloudJobSubmissionService,
   ComputeResourceService: ServiceFactory.service("ComputeResources"),
   CredentialSummaryService: ServiceFactory.service("CredentialSummaries"),
+  DataProductService: ServiceFactory.service("DataProducts"),
   ExperimentSearchService: ServiceFactory.service("ExperimentSearch"),
   ExperimentService: ServiceFactory.service("Experiments"),
   FullExperimentService,
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js
index e3adc09..9e7a7c8 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/service_config.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/service_config.js
@@ -4,6 +4,7 @@ import ApplicationModule from "./models/ApplicationModule";
 import BatchQueue from "./models/BatchQueue";
 import ComputeResourceDescription from "./models/ComputeResourceDescription";
 import CredentialSummary from "./models/CredentialSummary";
+import DataProduct from "./models/DataProduct";
 import Experiment from "./models/Experiment";
 import ExperimentSummary from "./models/ExperimentSummary";
 import GatewayResourceProfile from "./models/GatewayResourceProfile";
@@ -184,6 +185,15 @@ export default {
     ],
     modelClass: CredentialSummary
   },
+  DataProducts: {
+    url: "/api/data-products/",
+    viewSet: [
+      {
+        name: "retrieve"
+      }
+    ],
+    modelClass: DataProduct
+  },
   Experiments: {
     url: "/api/experiments/",
     viewSet: [
diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py
index eb81baf..064bd1c 100644
--- a/django_airavata/apps/api/urls.py
+++ b/django_airavata/apps/api/urls.py
@@ -41,6 +41,9 @@ router.register(r'storage-preferences',
                 views.StoragePreferenceViewSet,
                 base_name='storage-preference')
 router.register(r'parsers', views.ParserViewSet, base_name='parser')
+router.register(r'data-products',
+                views.DataProductViewSet,
+                base_name='data-product')
 
 app_name = 'django_airavata_api'
 urlpatterns = [
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 52a959d..0ff7e61 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -159,15 +159,7 @@ class ExperimentViewSet(APIBackedViewSet):
         experiment = serializer.save(
             gatewayId=self.gateway_id,
             userName=self.username)
-        experiment.userConfigurationData.storageId = \
-            settings.GATEWAY_DATA_STORE_RESOURCE_ID
-        # Set the experimentDataDir
-        project = self.request.airavata_client.getProject(
-            self.authz_token, experiment.projectId)
-        exp_dir = datastore.get_experiment_dir(self.username,
-                                               project.name,
-                                               experiment.experimentName)
-        experiment.userConfigurationData.experimentDataDir = exp_dir
+        self._set_storage_id_and_data_dir(experiment)
         experiment_id = self.request.airavata_client.createExperiment(
             self.authz_token, self.gateway_id, experiment)
         experiment.experimentId = experiment_id
@@ -176,8 +168,28 @@ class ExperimentViewSet(APIBackedViewSet):
         experiment = serializer.save(
             gatewayId=self.gateway_id,
             userName=self.username)
+        # The project or exp name may have changed, so update the exp data dir
+        self._set_storage_id_and_data_dir(experiment)
         self.request.airavata_client.updateExperiment(
             self.authz_token, experiment.experimentId, experiment)
+        # Process experiment._removed_input_files, removing them from storage
+        for removed_input_file in experiment._removed_input_files:
+            data_product = self.request.airavata_client.getDataProduct(
+                self.authz_token, removed_input_file)
+            datastore.delete(data_product)
+
+    def _set_storage_id_and_data_dir(self, experiment):
+        # Storage ID
+        experiment.userConfigurationData.storageId = \
+            settings.GATEWAY_DATA_STORE_RESOURCE_ID
+        # Create experiment dir and set it on model
+        project = self.request.airavata_client.getProject(
+            self.authz_token, experiment.projectId)
+        exp_dir = datastore.get_experiment_dir(self.username,
+                                               project.name,
+                                               experiment.experimentName)
+        experiment.userConfigurationData.experimentDataDir = exp_dir
+
 
     @detail_route(methods=['post'])
     def launch(self, request, experiment_id=None):
@@ -611,6 +623,17 @@ class LocalDataMovementView(APIView):
                 instance=data_movement).data)
 
 
+class DataProductViewSet(mixins.RetrieveModelMixin,
+                         GenericAPIBackedViewSet):
+    serializer_class = serializers.DataProductSerializer
+    lookup_field = 'product_uri'
+    lookup_value_regex = '.*'
+
+    def get_instance(self, lookup_value):
+        return self.request.airavata_client.getDataProduct(
+            self.request.authz_token, lookup_value)
+
+
 @login_required
 def upload_input_file(request):
     try:
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue
b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue
new file mode 100644
index 0000000..4f12b4f
--- /dev/null
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/DataProductViewer.vue
@@ -0,0 +1,40 @@
+<template>
+
+  <span v-if="dataProduct.downloadURL">
+    <a :href="dataProduct.downloadURL">
+      <i class="fa fa-download"></i>
+      {{ filename }}
+    </a>
+  </span>
+  <span v-else>{{ filename }}</span>
+</template>
+
+<script>
+import { models } from "django-airavata-api";
+export default {
+  name: "data-product-viewer",
+  props: {
+    dataProduct: {
+      type: models.DataProduct,
+      required: true
+    },
+    inputFile: {
+      type: Boolean,
+      default: false
+    }
+  },
+  computed: {
+    filename() {
+      if (this.inputFile) {
+        // productName captures the user provided name of the file, which may
+        // not match the name of the file on the storage system (for example,
+        // because of file name collision)
+        return this.dataProduct.productName;
+      } else {
+        return this.dataProduct.filename;
+      }
+    }
+  }
+};
+</script>
+
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
index af529f4..9b2892d 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentSummary.vue
@@ -33,15 +33,7 @@
                 <tr>
                   <th scope="row">Outputs</th>
                   <td>
-                    <template v-for="output in localFullExperiment.outputDataProducts">
-                      <span v-if="output.downloadURL" :key="output.productUri">
-                        <a :href="output.downloadURL">
-                          <i class="fa fa-download"></i>
-                          {{ output.filename }}
-                        </a>
-                      </span>
-                      <span v-else :key="output.productUri">{{ output.filename }}</span>
-                    </template>
+                    <data-product-viewer v-for="output in localFullExperiment.outputDataProducts"
:data-product="output" class="data-product" :key="output.productUri"/>
                   </td>
                 </tr>
                 <!-- Going to leave this out for now -->
@@ -130,15 +122,8 @@
                 <tr>
                   <th scope="row">Inputs</th>
                   <td>
-                    <template v-for="input in localFullExperiment.inputDataProducts">
-                      <span v-if="input.downloadURL" :key="input.productUri">
-                        <a :href="input.downloadURL">
-                          <i class="fa fa-download"></i>
-                          {{ input.filename }}
-                        </a>
-                      </span>
-                      <span v-else :key="input.productUri">{{ input.filename }}</span>
-                    </template>
+                    <data-product-viewer v-for="input in localFullExperiment.inputDataProducts"
+                      :data-product="input" :input-file="true" class="data-product" :key="input.productUri"/>
                   </td>
                 </tr>
                 <tr>
@@ -157,6 +142,7 @@
 
 <script>
 import { models, services } from "django-airavata-api";
+import DataProductViewer from "./DataProductViewer.vue";
 
 import moment from "moment";
 
@@ -177,7 +163,9 @@ export default {
       localFullExperiment: this.fullExperiment.clone()
     };
   },
-  components: {},
+  components: {
+    DataProductViewer,
+  },
   computed: {
     creationTime: function() {
       return moment(this.localFullExperiment.experiment.creationTime).fromNow();
@@ -224,5 +212,8 @@ export default {
 };
 </script>
 
-<style>
+<style scoped>
+.data-product + .data-product {
+  margin-left: 0.5em;
+}
 </style>
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
index 964aa9f..9f5b496 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
@@ -1,14 +1,74 @@
 <template>
-    <b-form-file :id="id" v-model="data"
-        :placeholder="experimentInput.userFriendlyDescription"
-        :state="componentValidState"
-        @input="valueChanged"/>
+  <div>
+    <div
+      class="row"
+      v-if="isDataProductURI && dataProduct"
+    >
+      <div class="col mr-auto">
+        <data-product-viewer :data-product="dataProduct" :input-file="true"/>
+      </div>
+      <div class="col-auto">
+        <delete-link @delete="deleteDataProduct">
+          Are you sure you want to delete input file {{ dataProduct.filename }}?
+        </delete-link>
+      </div>
+    </div>
+    <div class="row">
+      <div class="col">
+
+        <b-form-file
+          :id="id"
+          v-model="data"
+          v-if="!isDataProductURI"
+          :placeholder="experimentInput.userFriendlyDescription"
+          :state="componentValidState"
+          @input="valueChanged"
+        />
+      </div>
+    </div>
+  </div>
 </template>
 
 <script>
-import {InputEditorMixin} from 'django-airavata-workspace-plugin-api'
+import { services } from "django-airavata-api";
+import { InputEditorMixin } from "django-airavata-workspace-plugin-api";
+import DataProductViewer from "../DataProductViewer.vue";
+import { components } from "django-airavata-common-ui";
+
 export default {
-    name: 'file-input-editor',
-    mixins: [InputEditorMixin],
-}
+  name: "file-input-editor",
+  mixins: [InputEditorMixin],
+  components: {
+    DataProductViewer,
+    "delete-link": components.DeleteLink
+  },
+  computed: {
+    isDataProductURI() {
+      // Just assume that if the value is a string then it's a data product URL
+      return this.value && typeof this.value === "string";
+    }
+  },
+  data() {
+    return {
+      dataProduct: null
+    };
+  },
+  created() {
+    if (this.isDataProductURI) {
+      this.loadDataProduct(this.value);
+    }
+  },
+  methods: {
+    loadDataProduct(dataProductURI) {
+      services.DataProductService.retrieve({ lookup: dataProductURI }).then(
+        dataProduct => (this.dataProduct = dataProduct)
+      );
+    },
+    deleteDataProduct() {
+      // Just null out the 'data' field. Backend will delete the file from storage
+      this.data = null;
+      this.valueChanged();
+    }
+  }
+};
 </script>
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
index a90d214..a2b9f2a 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/EditExperimentContainer.vue
@@ -17,7 +17,7 @@ import ExperimentEditor from "../components/experiment/ExperimentEditor.vue";
 import moment from "moment";
 
 export default {
-  name: "create-experiment-container",
+  name: "edit-experiment-container",
   props: {
     experimentId: {
       type: String,


Mime
View raw message