climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omk...@apache.org
Subject [04/16] climate git commit: CLIMATE-769 Create data source input for NASA JPL PO.DAAC
Date Sat, 27 Aug 2016 03:42:26 GMT
CLIMATE-769 Create data source input for NASA JPL PO.DAAC


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/a847e1b6
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/a847e1b6
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/a847e1b6

Branch: refs/heads/master
Commit: a847e1b645a0474f5fd01124fac55a5d8be2fec1
Parents: 116d16f
Author: Omkar20895 <omkarreddy2008@gmail.com>
Authored: Thu Aug 11 17:45:58 2016 +0530
Committer: Omkar20895 <omkarreddy2008@gmail.com>
Committed: Thu Aug 11 17:45:58 2016 +0530

----------------------------------------------------------------------
 ocw/data_source/podaac.py | 71 ++++++++++++++++++++-------------------
 ocw/tests/test_podaac.py  | 76 ++++++++++++++++++++++--------------------
 2 files changed, 77 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/a847e1b6/ocw/data_source/podaac.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/podaac.py b/ocw/data_source/podaac.py
index 1eb0195..47a5409 100644
--- a/ocw/data_source/podaac.py
+++ b/ocw/data_source/podaac.py
@@ -15,14 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import sys
-sys.path.append('/Users/omkar/Documents/Git/podaacpy/podaac')
-from podaac_data_source import Podaac 
+from podaac_data_source import Podaac
 import numpy as np
 from ocw.dataset import Dataset
 from netCDF4 import Dataset as netcdf_dataset
 from netcdftime import utime
-import os, urllib
+import os
+import urllib
 import xml.etree.ElementTree as ET
 
 
@@ -45,14 +44,13 @@ def _convert_times_to_datetime(time):
     return [parsed_time.num2date(x) for x in time[:]]
 
 
-
-def load_dataset(variable ,datasetId='', datasetShortName='', name=''):
+def load_dataset(variable, datasetId='', datasetShortName='', name=''):
     '''Loads a Dataset from PODAAC
 
     :param variable: The name of the variable to read from the dataset.
     :type variable: :mod:`string`
 
-   	:param datasetId: dataset persistent ID. datasetId or \
+        :param datasetId: dataset persistent ID. datasetId or \
         shortName is required for a granule search. Example: \
         PODAAC-ASOP2-25X01
     :type datasetId: :mod:`string` 
@@ -70,41 +68,46 @@ def load_dataset(variable ,datasetId='', datasetShortName='', name=''):
 
     :raises: ServerError
     '''
-    #Downloading the dataset using podaac toolkit
-	podaac = Podaac()
-	path = os.path.dirname(os.path.abspath(__file__))
-	granuleName = podaac.extract_l4_granule(datasetId=datasetId, shortName=datasetShortName,
path=path)
-	path = path+'/'+granuleName
-	d = netcdf_dataset(path, mode='r')
-	dataset = d.variables[variable]
+    # Downloading the dataset using podaac toolkit
+        podaac = Podaac()
+        path = os.path.dirname(os.path.abspath(__file__))
+        granuleName = podaac.extract_l4_granule(
+            datasetId=datasetId, shortName=datasetShortName, path=path)
+        path = path + '/' + granuleName
+        d = netcdf_dataset(path, mode='r')
+        dataset = d.variables[variable]
 
     # By convention, but not by standard, if the dimensions exist, they will be in the order:
     # time (t), altitude (z), latitude (y), longitude (x)
     # but conventions aren't always followed and all dimensions aren't always present so
     # see if we can make some educated deductions before defaulting to just pulling the first
three
     # columns.
-	temp_dimensions = map(lambda x:x.lower(),dataset.dimensions)
-	dataset_dimensions = dataset.dimensions
-	time = dataset_dimensions[temp_dimensions.index('time') if 'time' in temp_dimensions else
0]
-	lat = dataset_dimensions[temp_dimensions.index('lat') if 'lat' in temp_dimensions else 1]
-	lon = dataset_dimensions[temp_dimensions.index('lon') if 'lon' in temp_dimensions else 2]
+        temp_dimensions = map(lambda x: x.lower(), dataset.dimensions)
+        dataset_dimensions = dataset.dimensions
+        time = dataset_dimensions[temp_dimensions.index(
+            'time') if 'time' in temp_dimensions else 0]
+        lat = dataset_dimensions[temp_dimensions.index(
+            'lat') if 'lat' in temp_dimensions else 1]
+        lon = dataset_dimensions[temp_dimensions.index(
+            'lon') if 'lon' in temp_dimensions else 2]
 
     # Time is given to us in some units since an epoch. We need to convert
     # these values to datetime objects. Note that we use the main object's
     # time object and not the dataset specific reference to it. We need to
     # grab the 'units' from it and it fails on the dataset specific object.
-	times = np.array(_convert_times_to_datetime(d[time]))
-	lats = np.array(d.variables[lat][:])
-	lons = np.array(d.variables[lon][:])
-	values = np.array(dataset[:])
-	origin = {
-        'source' : 'PO.DAAC',
-        'url' : 'podaac.jpl.nasa.gov/ws'
-    }
-
-    # Removing the downloaded temporary granule before creating the OCW dataset. 
-	d.close()
-	path = os.path.join(os.path.dirname(__file__), granuleName)
-	os.remove(path)
-
-	return Dataset(lats, lons, times, values, variable, name=name, origin=origin)
+        times = np.array(_convert_times_to_datetime(d[time]))
+        lats = np.array(d.variables[lat][:])
+        lons = np.array(d.variables[lon][:])
+        values = np.array(dataset[:])
+        origin = {
+            'source': 'PO.DAAC',
+            'url': 'podaac.jpl.nasa.gov/ws'
+        }
+
+    # Removing the downloaded temporary granule before creating the OCW
+    # dataset.
+        d.close()
+        path = os.path.join(os.path.dirname(__file__), granuleName)
+        os.remove(path)
+
+        return Dataset(lats, lons, times, values, variable, name=name, origin=origin)

http://git-wip-us.apache.org/repos/asf/climate/blob/a847e1b6/ocw/tests/test_podaac.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_podaac.py b/ocw/tests/test_podaac.py
index d0c2808..f696d02 100644
--- a/ocw/tests/test_podaac.py
+++ b/ocw/tests/test_podaac.py
@@ -17,45 +17,49 @@
 
 
 import ocw.data_source.podaac as podaac
-import unittest, os
+import unittest
+import os
 from ocw.dataset import Dataset
 
+
 class TestPodaacDataSource(unittest.TestCase):
-	@classmethod
-	def setUpClass(cls):
-		cls.datasetId = 'PODAAC-GHCMC-4FM02'
-		cls.datasetShortName = 'CMC0.2deg-CMC-L4-GLOB-v2.0'
-		cls.variable = 'sea_ice_fraction'
-		cls.name = 'PO.DAAC_test_dataset'
-		cls.file_path = os.path.dirname(os.path.abspath(__file__))
-		cls.format = '.nc'
-		cls.dataset = podaac.load_dataset(cls.variable, cls.datasetId, cls.datasetShortName, cls.name)

-
-	def test_is_dataset(self):
-		self.assertTrue(isinstance(self.dataset, Dataset))
-	
-	def test_dataset_lats(self):
-		self.assertEquals(len(self.dataset.lats), 901)
-
-	def test_dataset_lons(self):
-		self.assertEquals(len(self.dataset.lons), 1800)
-
-	def test_dataset_times(self):
-		self.assertEquals(len(self.dataset.times), 1)
-
-	def test_dataset_values(self):
-		self.assertEquals(len(self.dataset.values),1)
-
-	def test_valid_date_conversion(self):
-        start = dt.datetime(2006, 6, 7, 12)
-        self.assertTrue(start == self.dataset.times[0])
-
-	def test_dataset_origin(self):
-		self.assertEquals(self.dataset.origin['source'], 'PO.DAAC')
-		self.assertEquals(self.dataset.origin['url'], 'podaac.jpl.nasa.gov/ws')
-
-	def test_custom_name(self):
-		self.assertEquals(self.dataset.name, self.name)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.datasetId = 'PODAAC-GHCMC-4FM02'
+        cls.datasetShortName = 'CMC0.2deg-CMC-L4-GLOB-v2.0'
+        cls.variable = 'sea_ice_fraction'
+        cls.name = 'PO.DAAC_test_dataset'
+        cls.file_path = os.path.dirname(os.path.abspath(__file__))
+        cls.format = '.nc'
+        cls.dataset = podaac.load_dataset(
+            cls.variable, cls.datasetId, cls.datasetShortName, cls.name)
+
+    def test_is_dataset(self):
+        self.assertTrue(isinstance(self.dataset, Dataset))
+
+    def test_dataset_lats(self):
+        self.assertEquals(len(self.dataset.lats), 901)
+
+    def test_dataset_lons(self):
+        self.assertEquals(len(self.dataset.lons), 1800)
+
+    def test_dataset_times(self):
+        self.assertEquals(len(self.dataset.times), 1)
+
+    def test_dataset_values(self):
+        self.assertEquals(len(self.dataset.values), 1)
+
+    def test_valid_date_conversion(self):
+    start = dt.datetime(2006, 6, 7, 12)
+    self.assertTrue(start == self.dataset.times[0])
+
+    def test_dataset_origin(self):
+        self.assertEquals(self.dataset.origin['source'], 'PO.DAAC')
+        self.assertEquals(self.dataset.origin['url'], 'podaac.jpl.nasa.gov/ws')
+
+    def test_custom_name(self):
+        self.assertEquals(self.dataset.name, self.name)
 
 
 if __name__ == '__main__':


Mime
View raw message