climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jo...@apache.org
Subject [1/2] git commit: CLIMATE-386 - Add functionality to output a dataset as NetCDF
Date Thu, 03 Apr 2014 02:25:16 GMT
Repository: climate
Updated Branches:
  refs/heads/master 7bde9e126 -> 82396cf4e


CLIMATE-386 - Add functionality to output a dataset as NetCDF

- Add dataset_processor.write_netcdf for outputting a Dataset object as
  a NetCDF4 file.
- Add unit tests for write_netcdf functionality.


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/97269f40
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/97269f40
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/97269f40

Branch: refs/heads/master
Commit: 97269f40ebf30c4413a0300a0ec743e507fd2737
Parents: c9558e0
Author: Michael Joyce <joyce@apache.org>
Authored: Sat Mar 29 16:31:02 2014 -0700
Committer: Michael Joyce <joyce@apache.org>
Committed: Sat Mar 29 16:31:02 2014 -0700

----------------------------------------------------------------------
 ocw/dataset_processor.py            | 48 ++++++++++++++++++++++++++++++++
 ocw/tests/test_dataset_processor.py | 25 +++++++++++++++++
 2 files changed, 73 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/97269f40/ocw/dataset_processor.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_processor.py b/ocw/dataset_processor.py
index e9adba2..72c0172 100644
--- a/ocw/dataset_processor.py
+++ b/ocw/dataset_processor.py
@@ -23,6 +23,7 @@ import numpy.ma as ma
 import scipy.interpolate
 import scipy.ndimage
 from scipy.ndimage import map_coordinates
+import netCDF4
 
 import logging
 
@@ -210,6 +211,53 @@ def normalize_dataset_datetimes(dataset, timestep):
         dataset.name
     )
 
+def write_netcdf(dataset, path, compress=True):
+    ''' Write a dataset to a NetCDF file.
+
+    :param dataset: The dataset to write.
+    :type dataset: ocw.dataset.Dataset
+
+    :param path: The output file path.
+    :type path: string
+    '''
+    out_file = netCDF4.Dataset(path, 'w', format='NETCDF4')
+
+    # Set attribute lenghts
+    lat_len = len(dataset.lats)
+    lon_len = len(dataset.lons)
+    time_len = len(dataset.times)
+
+    # Create attribute dimensions
+    lat_dim = out_file.createDimension('lat', lat_len)
+    lon_dim = out_file.createDimension('lon', lon_len)
+    time_dim = out_file.createDimension('time', time_len)
+
+    # Create variables
+    lats = out_file.createVariable('lat', 'f8', ('lat',), zlib=compress)
+    lons = out_file.createVariable('lon', 'f8', ('lon',), zlib=compress)
+    times = out_file.createVariable('time', 'f8', ('time',), zlib=compress)
+
+    var_name = dataset.variable if dataset.variable else 'var'
+    values = out_file.createVariable(var_name,
+                                    'f8',
+                                    ('time', 'lat', 'lon'),
+                                    zlib=compress)
+
+    # Set the time variable units
+    # We don't deal with hourly/minutely/anything-less-than-a-day data so
+    # we can safely stick with a 'days since' offset here. Note that the
+    # NetCDF4 helper date2num doesn't support 'months' or 'years' instead
+    # of days.
+    times.units = "days since %s" % dataset.times[0]
+
+    # Store the dataset's values
+    lats[:] = dataset.lats
+    lons[:] = dataset.lons
+    times[:] = netCDF4.date2num(dataset.times, times.units)
+    values[:] = dataset.values
+
+    out_file.close()
+
 def _rcmes_normalize_datetimes(datetimes, timestep):
     """ Normalize Dataset datetime values.
 

http://git-wip-us.apache.org/repos/asf/climate/blob/97269f40/ocw/tests/test_dataset_processor.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_dataset_processor.py b/ocw/tests/test_dataset_processor.py
index a144d4a..f883ad9 100644
--- a/ocw/tests/test_dataset_processor.py
+++ b/ocw/tests/test_dataset_processor.py
@@ -17,8 +17,11 @@
 
 import unittest
 import datetime
+import os
+
 from ocw import dataset_processor as dp
 from ocw import dataset as ds
+from ocw.data_source import local
 import numpy as np
 import numpy.ma as ma
 
@@ -268,6 +271,28 @@ class TestFailingSubset(unittest.TestCase):
         with self.assertRaises(ValueError):
             dp.subset(self.subregion, self.target_dataset)
 
+class TestNetCDFWrite(unittest.TestCase):
+    def setUp(self):
+        self.ds = ten_year_monthly_dataset()
+        self.file_name = 'test.nc'
+
+    def tearDown(self):
+        if os.path.isfile(self.file_name):
+            os.remove(self.file_name)
+
+    def test_file_write(self):
+        dp.write_netcdf(self.ds, self.file_name)
+        self.assertTrue(os.path.isfile(self.file_name))
+
+    def test_that_file_contents_are_valid(self):
+        dp.write_netcdf(self.ds, self.file_name)
+        new_ds = local.load_file(self.file_name, self.ds.variable)
+
+        self.assertEqual(self.ds.variable, new_ds.variable)
+        self.assertTrue(np.array_equal(self.ds.lats, new_ds.lats))
+        self.assertTrue(np.array_equal(self.ds.lons, new_ds.lons))
+        self.assertTrue(np.array_equal(self.ds.times, new_ds.times))
+        self.assertTrue(np.array_equal(self.ds.values, new_ds.values))
 
 def ten_year_monthly_dataset():
     lats = np.array(range(-89, 90, 2))


Mime
View raw message