Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A949C200BB6 for ; Fri, 21 Oct 2016 04:46:43 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A7D11160AF2; Fri, 21 Oct 2016 02:46:43 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C8F55160AE0 for ; Fri, 21 Oct 2016 04:46:42 +0200 (CEST) Received: (qmail 95567 invoked by uid 500); 21 Oct 2016 02:46:42 -0000 Mailing-List: contact commits-help@climate.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@climate.apache.org Delivered-To: mailing list commits@climate.apache.org Received: (qmail 95558 invoked by uid 99); 21 Oct 2016 02:46:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Oct 2016 02:46:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9AC0CE0061; Fri, 21 Oct 2016 02:46:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lewismc@apache.org To: commits@climate.apache.org Message-Id: <947a1ff9f0e64a7c8f20c914290a36e9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: climate git commit: CLIMATE-875 Upgrade to Podaacpy 1.4.0 Date: Fri, 21 Oct 2016 02:46:41 +0000 (UTC) archived-at: Fri, 21 Oct 2016 02:46:43 -0000 Repository: climate Updated Branches: refs/heads/master 8cf223f2c -> 088706d82 CLIMATE-875 Upgrade to Podaacpy 1.4.0 Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/088706d8 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/088706d8 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/088706d8 Branch: refs/heads/master Commit: 088706d8209204869fbaf24d80c7cc7b7b9aeda3 Parents: 8cf223f Author: Lewis John McGibbney Authored: Thu Oct 20 19:08:23 2016 -0700 Committer: Lewis John McGibbney Committed: Thu Oct 20 19:08:23 2016 -0700 ---------------------------------------------------------------------- examples/podaac_integration_example.py | 4 ++-- ocw/data_source/podaac_datasource.py | 36 ++++++++++++++++++++++++++--- ocw/dataset_loader.py | 2 +- ocw/tests/subset.json | 12 ++++++++++ ocw/tests/test_podaac.py | 12 +++++++++- 5 files changed, 59 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/examples/podaac_integration_example.py ---------------------------------------------------------------------- diff --git a/examples/podaac_integration_example.py b/examples/podaac_integration_example.py index da90373..a06fc94 100644 --- a/examples/podaac_integration_example.py +++ b/examples/podaac_integration_example.py @@ -25,8 +25,8 @@ variable = 'uwnd' name = 'PO.DAAC_test_dataset' OUTPUT_PLOT = "ccmp_temporal_std" """ Step 1: Load Local NetCDF Files into OCW Dataset Objects """ -print("Loading %s dataset into an OCW dataset objects" % datasetShortName) -ccmp_dataset = podaac.load_dataset( +print("Loading %s dataset short name into a OCW dataset object." % datasetId) +ccmp_dataset = podaac.load_level4_granule( variable=variable, datasetId=datasetId, name=name) print("CCMP_Dataset.values shape: (times, lats, lons) - %s \n" % (ccmp_dataset.values.shape,)) http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/data_source/podaac_datasource.py ---------------------------------------------------------------------- diff --git a/ocw/data_source/podaac_datasource.py b/ocw/data_source/podaac_datasource.py index 4a08046..6feb02b 100644 --- a/ocw/data_source/podaac_datasource.py +++ b/ocw/data_source/podaac_datasource.py @@ -16,6 +16,7 @@ # under the License. from podaac.podaac import Podaac +from podaac.podaac_utils import PodaacUtils import numpy as np from ocw.dataset import Dataset from netCDF4 import Dataset as netcdf_dataset @@ -42,9 +43,38 @@ def convert_times_to_datetime(time): return [parsed_time.num2date(x) for x in time[:]] -def load_dataset(variable, datasetId='', name=''): - '''Loads a Dataset from PODAAC +def list_available_extract_granule_dataset_ids(): + '''Convenience function which returns an up-to-date \ + list of available granule dataset id's which can be \ + used in the granule extraction service. + :returns: a comma-seperated list of granule dataset id's. + ''' + podaac_utils = PodaacUtils() + return podaac_utils.list_available_extract_granule_dataset_ids() + +def subset_granule(input_file_path=''): + '''Subset Granule service allows users to Submit subset jobs. \ + Use of this service should be preceded by a Granule Search in \ + order to identify and generate a list of granules to be subsetted. + + :param input_file_path: path to a json file which contains the \ + the request that you want to send to PO.DAAC + :type input_file_path: :mod:`string` + + :returns: a token on successful request reception. This can be \ + further used to check the status of the request. + + ''' + podaac = Podaac() + status = podaac.subset_status(podaac.granule_subset(input_file_path)) + print("Granule subsetting initiated with request tracking token '%s'." % status) + while status is not "done": + print('...') + return status + +def load_level4_granule(variable, datasetId='', name=''): + '''Loads a Level4 gridded Dataset from PODAAC :param variable: The name of the variable to read from the dataset. :type variable: :mod:`string` @@ -80,7 +110,7 @@ def load_dataset(variable, datasetId='', name=''): # but conventions aren't always followed and all dimensions aren't always present so # see if we can make some educated deductions before defaulting to just pulling the first three # columns. - temp_dimensions = map(lambda x: x.lower(), dataset.dimensions) + temp_dimensions = list(map(lambda x: x.lower(), dataset.dimensions)) dataset_dimensions = dataset.dimensions time = dataset_dimensions[temp_dimensions.index( 'time') if 'time' in temp_dimensions else 0] http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/dataset_loader.py ---------------------------------------------------------------------- diff --git a/ocw/dataset_loader.py b/ocw/dataset_loader.py index fdcbb1b..5a7f6ea 100644 --- a/ocw/dataset_loader.py +++ b/ocw/dataset_loader.py @@ -94,7 +94,7 @@ class DatasetLoader: 'local': local.load_multiple_files, 'local_split': local.load_dataset_from_multiple_netcdf_files, 'rcmed': rcmed.parameter_dataset, - 'podaac': podaac.load_dataset + 'podaac': podaac.load_level4_granule } # Exclude esgf and dap for python 3 until they are compatible http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/tests/subset.json ---------------------------------------------------------------------- diff --git a/ocw/tests/subset.json b/ocw/tests/subset.json new file mode 100644 index 0000000..c00891a --- /dev/null +++ b/ocw/tests/subset.json @@ -0,0 +1,12 @@ +{ + "email":"yournamem@emailaddress.com", + "query":[ + { + "datasetId":"PODAAC-ASOP2-25X01", + "granuleIds":["ascat_20160409_113000_metopa_49153_eps_o_250_2401_ovw.l2.nc"], + "bbox":"-180,-90,180,90", + "variables":["wvc_index","model_speed","model_dir","ice_prob","ice_age","wvc_quality_flag","wind_speed","wind_dir","bs_distance","lat","lon","time"], + "compact":false + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/climate/blob/088706d8/ocw/tests/test_podaac.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_podaac.py b/ocw/tests/test_podaac.py index 66afcb8..89bc143 100644 --- a/ocw/tests/test_podaac.py +++ b/ocw/tests/test_podaac.py @@ -32,8 +32,13 @@ class TestPodaacDataSource(unittest.TestCase): cls.name = 'PO.DAAC_test_dataset' cls.file_path = os.path.dirname(os.path.abspath(__file__)) cls.format = '.nc' - cls.dataset = podaac.load_dataset( + cls.dataset = podaac.load_level4_granule( cls.variable, cls.datasetId, cls.name) + #Until we can retrieve the subset data download link programmatically, + #we will need to skip this test. More information can be see at + #https://podaac.jpl.nasa.gov/forum/viewtopic.php?f=53&t=424&p=790 + #cls.json = 'subset.json' + #cls.granule_subset = podaac.subset_granule(cls.json) def test_is_dataset(self): self.assertTrue(isinstance(self.dataset, Dataset)) @@ -61,6 +66,11 @@ class TestPodaacDataSource(unittest.TestCase): def test_custom_name(self): self.assertEquals(self.dataset.name, self.name) + def test_granule_subset(self): + #Until we can retrieve the subset data download link programmatically, + #we will need to skip this test. More information can be see at + #https://podaac.jpl.nasa.gov/forum/viewtopic.php?f=53&t=424&p=790 + pass if __name__ == '__main__': unittest.main()