climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lewi...@apache.org
Subject climate git commit: CLIMATE-875 Upgrade to Podaacpy 1.4.0
Date Fri, 21 Oct 2016 02:46:41 GMT
Repository: climate
Updated Branches:
  refs/heads/master 8cf223f2c -> 088706d82


CLIMATE-875 Upgrade to Podaacpy 1.4.0


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/088706d8
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/088706d8
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/088706d8

Branch: refs/heads/master
Commit: 088706d8209204869fbaf24d80c7cc7b7b9aeda3
Parents: 8cf223f
Author: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Authored: Thu Oct 20 19:08:23 2016 -0700
Committer: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Committed: Thu Oct 20 19:08:23 2016 -0700

----------------------------------------------------------------------
 examples/podaac_integration_example.py |  4 ++--
 ocw/data_source/podaac_datasource.py   | 36 ++++++++++++++++++++++++++---
 ocw/dataset_loader.py                  |  2 +-
 ocw/tests/subset.json                  | 12 ++++++++++
 ocw/tests/test_podaac.py               | 12 +++++++++-
 5 files changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/examples/podaac_integration_example.py
----------------------------------------------------------------------
diff --git a/examples/podaac_integration_example.py b/examples/podaac_integration_example.py
index da90373..a06fc94 100644
--- a/examples/podaac_integration_example.py
+++ b/examples/podaac_integration_example.py
@@ -25,8 +25,8 @@ variable = 'uwnd'
 name = 'PO.DAAC_test_dataset'
 OUTPUT_PLOT = "ccmp_temporal_std"
 """ Step 1: Load Local NetCDF Files into OCW Dataset Objects """
-print("Loading %s dataset into an OCW dataset objects" % datasetShortName)
-ccmp_dataset = podaac.load_dataset(
+print("Loading %s dataset short name into a OCW dataset object." % datasetId)
+ccmp_dataset = podaac.load_level4_granule(
     variable=variable, datasetId=datasetId, name=name)
 print("CCMP_Dataset.values shape: (times, lats, lons) - %s \n" %
       (ccmp_dataset.values.shape,))

http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/data_source/podaac_datasource.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/podaac_datasource.py b/ocw/data_source/podaac_datasource.py
index 4a08046..6feb02b 100644
--- a/ocw/data_source/podaac_datasource.py
+++ b/ocw/data_source/podaac_datasource.py
@@ -16,6 +16,7 @@
 # under the License.
 
 from podaac.podaac import Podaac
+from podaac.podaac_utils import PodaacUtils
 import numpy as np
 from ocw.dataset import Dataset
 from netCDF4 import Dataset as netcdf_dataset
@@ -42,9 +43,38 @@ def convert_times_to_datetime(time):
     return [parsed_time.num2date(x) for x in time[:]]
 
 
-def load_dataset(variable, datasetId='', name=''):
-    '''Loads a Dataset from PODAAC
+def list_available_extract_granule_dataset_ids():
+    '''Convenience function which returns an up-to-date \
+        list of available granule dataset id's which can be \
+        used in the granule extraction service.
+    :returns: a comma-seperated list of granule dataset id's.
 
+    '''
+    podaac_utils = PodaacUtils()
+    return podaac_utils.list_available_extract_granule_dataset_ids()
+
+def subset_granule(input_file_path=''):
+    '''Subset Granule service allows users to Submit subset jobs. \
+        Use of this service should be preceded by a Granule Search in \
+        order to identify and generate a list of granules to be subsetted.
+
+    :param input_file_path: path to a json file which contains the \
+        the request that you want to send to PO.DAAC
+    :type input_file_path: :mod:`string`
+
+    :returns: a token on successful request reception. This can be \
+        further used to check the status of the request.
+
+    '''
+    podaac = Podaac()
+    status = podaac.subset_status(podaac.granule_subset(input_file_path))
+    print("Granule subsetting initiated with request tracking token '%s'." % status)
+    while status is not "done":
+        print('...')
+    return status
+
+def load_level4_granule(variable, datasetId='', name=''):
+    '''Loads a Level4 gridded Dataset from PODAAC
     :param variable: The name of the variable to read from the dataset.
     :type variable: :mod:`string`
 
@@ -80,7 +110,7 @@ def load_dataset(variable, datasetId='', name=''):
     # but conventions aren't always followed and all dimensions aren't always present so
     # see if we can make some educated deductions before defaulting to just pulling the first
three
     # columns.
-    temp_dimensions = map(lambda x: x.lower(), dataset.dimensions)
+    temp_dimensions = list(map(lambda x: x.lower(), dataset.dimensions))
     dataset_dimensions = dataset.dimensions
     time = dataset_dimensions[temp_dimensions.index(
         'time') if 'time' in temp_dimensions else 0]

http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/dataset_loader.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_loader.py b/ocw/dataset_loader.py
index fdcbb1b..5a7f6ea 100644
--- a/ocw/dataset_loader.py
+++ b/ocw/dataset_loader.py
@@ -94,7 +94,7 @@ class DatasetLoader:
             'local': local.load_multiple_files,
             'local_split': local.load_dataset_from_multiple_netcdf_files,
             'rcmed': rcmed.parameter_dataset,
-            'podaac': podaac.load_dataset
+            'podaac': podaac.load_level4_granule
         }
         
         # Exclude esgf and dap for python 3 until they are compatible

http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/tests/subset.json
----------------------------------------------------------------------
diff --git a/ocw/tests/subset.json b/ocw/tests/subset.json
new file mode 100644
index 0000000..c00891a
--- /dev/null
+++ b/ocw/tests/subset.json
@@ -0,0 +1,12 @@
+{
+    "email":"yournamem@emailaddress.com",
+    "query":[
+        {
+            "datasetId":"PODAAC-ASOP2-25X01",
+            "granuleIds":["ascat_20160409_113000_metopa_49153_eps_o_250_2401_ovw.l2.nc"],
+            "bbox":"-180,-90,180,90",
+            "variables":["wvc_index","model_speed","model_dir","ice_prob","ice_age","wvc_quality_flag","wind_speed","wind_dir","bs_distance","lat","lon","time"],
+            "compact":false
+        }
+    ]
+} 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/tests/test_podaac.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_podaac.py b/ocw/tests/test_podaac.py
index 66afcb8..89bc143 100644
--- a/ocw/tests/test_podaac.py
+++ b/ocw/tests/test_podaac.py
@@ -32,8 +32,13 @@ class TestPodaacDataSource(unittest.TestCase):
         cls.name = 'PO.DAAC_test_dataset'
         cls.file_path = os.path.dirname(os.path.abspath(__file__))
         cls.format = '.nc'
-        cls.dataset = podaac.load_dataset(
+        cls.dataset = podaac.load_level4_granule(
             cls.variable, cls.datasetId, cls.name)
+        #Until we can retrieve the subset data download link programmatically,
+        #we will need to skip this test. More information can be see at 
+        #https://podaac.jpl.nasa.gov/forum/viewtopic.php?f=53&t=424&p=790
+        #cls.json = 'subset.json'
+        #cls.granule_subset = podaac.subset_granule(cls.json)
 
     def test_is_dataset(self):
         self.assertTrue(isinstance(self.dataset, Dataset))
@@ -61,6 +66,11 @@ class TestPodaacDataSource(unittest.TestCase):
     def test_custom_name(self):
         self.assertEquals(self.dataset.name, self.name)
 
+    def test_granule_subset(self):
+        #Until we can retrieve the subset data download link programmatically,
+        #we will need to skip this test. More information can be see at 
+        #https://podaac.jpl.nasa.gov/forum/viewtopic.php?f=53&t=424&p=790
+        pass
 
 if __name__ == '__main__':
     unittest.main()


Mime
View raw message