climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jo...@apache.org
Subject [08/13] climate git commit: CLIMATE-581 - Add dataset export from evaluation object
Date Tue, 31 Mar 2015 14:59:27 GMT
CLIMATE-581 - Add dataset export from evaluation object

- Add a helper for exporting all dataset configuration information from
  an Evaluation object.
- Rename the existing single dataset export function so functions that
  export from an Evaluation object can share consistent naming.
- Update tests with new dataset export function name.
- Add tests for full dataset export from an Evaluation object.


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/b0e7b40c
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/b0e7b40c
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/b0e7b40c

Branch: refs/heads/master
Commit: b0e7b40c7ae49247c960de7c10f0532274402964
Parents: 33b36be
Author: Michael Joyce <joyce@apache.org>
Authored: Wed Mar 25 10:24:00 2015 -0700
Committer: Michael Joyce <joyce@apache.org>
Committed: Wed Mar 25 10:24:00 2015 -0700

----------------------------------------------------------------------
 ocw-config-runner/configuration_writer.py     |  27 ++++-
 ocw-config-runner/tests/test_config_writer.py | 133 ++++++++++++++++++++-
 2 files changed, 154 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/b0e7b40c/ocw-config-runner/configuration_writer.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/configuration_writer.py b/ocw-config-runner/configuration_writer.py
index 9e02a4a..497c7db 100644
--- a/ocw-config-runner/configuration_writer.py
+++ b/ocw-config-runner/configuration_writer.py
@@ -21,8 +21,31 @@ import logging
 logging.basicConfig()
 logger = logging.getLogger(__name__)
 
-def generate_dataset_information(dataset):
-    ''' Generates a dict of dataset information for export.
+def generate_dataset_information(evaluation):
+    ''' Generate dataset config file output for a given Evaluation object.
+    
+    :param evaluation: The evaluation object from which to extract metrics.
+    :type evaluation: :class:`evaluation.Evaluation`
+
+    :returns: A :func:`dict` of dataset configuration information for export
+        to a configuration file.
+    :rtype: :func:`dict`
+    '''
+    datasets = {}
+
+    if evaluation.ref_dataset:
+        datasets['reference'] = generate_dataset_config(evaluation.ref_dataset)
+
+    if len(evaluation.target_datasets) > 0:
+        datasets['targets'] = [
+            generate_dataset_config(target)
+            for target in evaluation.target_datasets
+        ]
+
+    return datasets
+
+def generate_dataset_config(dataset):
+    ''' Generate dataset config file output for a given Dataset object.
 
     :param dataset: The dataset from which to extract configuration
         information.

http://git-wip-us.apache.org/repos/asf/climate/blob/b0e7b40c/ocw-config-runner/tests/test_config_writer.py
----------------------------------------------------------------------
diff --git a/ocw-config-runner/tests/test_config_writer.py b/ocw-config-runner/tests/test_config_writer.py
index 1518d96..da1657d 100644
--- a/ocw-config-runner/tests/test_config_writer.py
+++ b/ocw-config-runner/tests/test_config_writer.py
@@ -59,7 +59,7 @@ class TestLocalDatasetExportGeneration(unittest.TestCase):
             name=self.name
         )
 
-        self.exported_info = writer.generate_dataset_information(self.dataset)
+        self.exported_info = writer.generate_dataset_config(self.dataset)
 
     def test_proper_data_source_export(self):
         self.assertTrue('data_source' in self.exported_info)
@@ -125,7 +125,7 @@ class TestRCMEDDatasetExportGeneration(unittest.TestCase):
             name=self.name
         )
 
-        self.exported_info = writer.generate_dataset_information(self.dataset)
+        self.exported_info = writer.generate_dataset_config(self.dataset)
 
     def test_proper_data_source_export(self):
         self.assertTrue('data_source' in self.exported_info)
@@ -195,7 +195,7 @@ class TestESGFDatasetExportGeneration(unittest.TestCase):
             name=self.name
         )
 
-        self.exported_info = writer.generate_dataset_information(self.dataset)
+        self.exported_info = writer.generate_dataset_config(self.dataset)
 
     def test_proper_data_source_export(self):
         self.assertTrue('data_source' in self.exported_info)
@@ -252,7 +252,7 @@ class TestDAPDatasetExportGeneration(unittest.TestCase):
             name=self.name
         )
 
-        self.exported_info = writer.generate_dataset_information(self.dataset)
+        self.exported_info = writer.generate_dataset_config(self.dataset)
 
     def test_proper_data_source_export(self):
         self.assertTrue('data_source' in self.exported_info)
@@ -272,6 +272,122 @@ class TestDAPDatasetExportGeneration(unittest.TestCase):
                          self.units)
 
 
+class TestDatasetExportFromEvaluation(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.lats = np.array([10, 12, 14, 16, 18])
+        self.lons = np.array([100, 102, 104, 106, 108])
+        self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)])
+        flat_array = np.array(range(300))
+        self.values = flat_array.reshape(12, 5, 5)
+        self.variable = 'var'
+        self.units = 'units'
+        self.name = 'name'
+
+        self.local_origin = {
+            'source': 'local',
+            'path': '/a/fake/path.nc',
+            'lat_name': 'a lat name',
+            'lon_name': 'a lon name',
+            'time_name': 'a time name',
+            'elevation_index': 2
+        }
+        
+        self.rcmed_origin = {
+            'source': 'rcmed',
+            'dataset_id': 4,
+            'parameter_id': 14
+        }
+
+        self.esgf_origin = {
+            'source': 'esgf',
+            'dataset_id': 'esgf dataset id',
+            'variable': 'var'
+        }
+
+        self.dap_origin = {
+            'source': 'dap',
+            'url': 'a fake url',
+        }
+
+        self.local_ds = Dataset(
+            self.lats,
+            self.lons,
+            self.times,
+            self.values,
+            variable=self.variable,
+            units=self.units,
+            name=self.name,
+            origin=self.local_origin
+        )
+
+        self.rcmed_ds = Dataset(
+            self.lats,
+            self.lons,
+            self.times,
+            self.values,
+            variable=self.variable,
+            units=self.units,
+            name=self.name,
+            origin=self.rcmed_origin
+        )
+        
+        self.esgf_ds = Dataset(
+            self.lats,
+            self.lons,
+            self.times,
+            self.values,
+            variable=self.variable,
+            units=self.units,
+            name=self.name,
+            origin=self.esgf_origin
+        )
+
+        self.dap_ds = Dataset(
+            self.lats,
+            self.lons,
+            self.times,
+            self.values,
+            variable=self.variable,
+            units=self.units,
+            name=self.name,
+            origin=self.dap_origin
+        )
+
+        self.evaluation = Evaluation(
+            self.local_ds,
+            [self.rcmed_ds, self.esgf_ds, self.dap_ds],
+            []
+        )
+
+    def test_contains_only_reference_dataset(self):
+        new_eval = Evaluation(self.local_ds, [], [])
+        out = writer.generate_dataset_information(new_eval)
+
+        self.assertTrue('reference' in out)
+        self.assertTrue('targets' not in out)
+
+    def test_contains_only_target_datasets(self):
+        new_eval = Evaluation(None, [self.local_ds], [])
+        out = writer.generate_dataset_information(new_eval)
+
+        self.assertTrue('reference' not in out)
+        self.assertTrue('targets' in out)
+
+    def test_proper_reference_dataset_export(self):
+        out = writer.generate_dataset_information(self.evaluation)
+
+        self.assertTrue('reference' in out)
+        self.assertTrue(out['reference']['data_source'] == 'local')
+
+    def test_proper_target_datasets_export(self):
+        out = writer.generate_dataset_information(self.evaluation)
+
+        self.assertTrue('targets' in out)
+        self.assertTrue(type(out['targets']) == type(list()))
+        self.assertTrue(len(out['targets']) == 3)
+
+
 class TestMetricExportGeneration(unittest.TestCase):
     @classmethod
     def setUpClass(self):
@@ -450,3 +566,12 @@ class TestEvaluationSettingsGeneration(unittest.TestCase):
         self.assertEqual(ds_lon_max, subset[3])
         self.assertEquals(str(start), subset[4])
         self.assertEquals(str(end), subset[5])
+
+
+class FullExportTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        pass
+
+    def test_full_export(self):
+        pass


Mime
View raw message