CLIMATE-581 - Add dataset export from evaluation object - Add a helper for exporting all dataset configuration information from an Evaluation object. - Rename the existing single dataset export function so functions that export from an Evaluation object can share consistent naming. - Update tests with new dataset export function name. - Add tests for full dataset export from an Evaluation object. Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/b0e7b40c Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/b0e7b40c Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/b0e7b40c Branch: refs/heads/master Commit: b0e7b40c7ae49247c960de7c10f0532274402964 Parents: 33b36be Author: Michael Joyce Authored: Wed Mar 25 10:24:00 2015 -0700 Committer: Michael Joyce Committed: Wed Mar 25 10:24:00 2015 -0700 ---------------------------------------------------------------------- ocw-config-runner/configuration_writer.py | 27 ++++- ocw-config-runner/tests/test_config_writer.py | 133 ++++++++++++++++++++- 2 files changed, 154 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/b0e7b40c/ocw-config-runner/configuration_writer.py ---------------------------------------------------------------------- diff --git a/ocw-config-runner/configuration_writer.py b/ocw-config-runner/configuration_writer.py index 9e02a4a..497c7db 100644 --- a/ocw-config-runner/configuration_writer.py +++ b/ocw-config-runner/configuration_writer.py @@ -21,8 +21,31 @@ import logging logging.basicConfig() logger = logging.getLogger(__name__) -def generate_dataset_information(dataset): - ''' Generates a dict of dataset information for export. +def generate_dataset_information(evaluation): + ''' Generate dataset config file output for a given Evaluation object. + + :param evaluation: The evaluation object from which to extract metrics. + :type evaluation: :class:`evaluation.Evaluation` + + :returns: A :func:`dict` of dataset configuration information for export + to a configuration file. + :rtype: :func:`dict` + ''' + datasets = {} + + if evaluation.ref_dataset: + datasets['reference'] = generate_dataset_config(evaluation.ref_dataset) + + if len(evaluation.target_datasets) > 0: + datasets['targets'] = [ + generate_dataset_config(target) + for target in evaluation.target_datasets + ] + + return datasets + +def generate_dataset_config(dataset): + ''' Generate dataset config file output for a given Dataset object. :param dataset: The dataset from which to extract configuration information. http://git-wip-us.apache.org/repos/asf/climate/blob/b0e7b40c/ocw-config-runner/tests/test_config_writer.py ---------------------------------------------------------------------- diff --git a/ocw-config-runner/tests/test_config_writer.py b/ocw-config-runner/tests/test_config_writer.py index 1518d96..da1657d 100644 --- a/ocw-config-runner/tests/test_config_writer.py +++ b/ocw-config-runner/tests/test_config_writer.py @@ -59,7 +59,7 @@ class TestLocalDatasetExportGeneration(unittest.TestCase): name=self.name ) - self.exported_info = writer.generate_dataset_information(self.dataset) + self.exported_info = writer.generate_dataset_config(self.dataset) def test_proper_data_source_export(self): self.assertTrue('data_source' in self.exported_info) @@ -125,7 +125,7 @@ class TestRCMEDDatasetExportGeneration(unittest.TestCase): name=self.name ) - self.exported_info = writer.generate_dataset_information(self.dataset) + self.exported_info = writer.generate_dataset_config(self.dataset) def test_proper_data_source_export(self): self.assertTrue('data_source' in self.exported_info) @@ -195,7 +195,7 @@ class TestESGFDatasetExportGeneration(unittest.TestCase): name=self.name ) - self.exported_info = writer.generate_dataset_information(self.dataset) + self.exported_info = writer.generate_dataset_config(self.dataset) def test_proper_data_source_export(self): self.assertTrue('data_source' in self.exported_info) @@ -252,7 +252,7 @@ class TestDAPDatasetExportGeneration(unittest.TestCase): name=self.name ) - self.exported_info = writer.generate_dataset_information(self.dataset) + self.exported_info = writer.generate_dataset_config(self.dataset) def test_proper_data_source_export(self): self.assertTrue('data_source' in self.exported_info) @@ -272,6 +272,122 @@ class TestDAPDatasetExportGeneration(unittest.TestCase): self.units) +class TestDatasetExportFromEvaluation(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.name = 'name' + + self.local_origin = { + 'source': 'local', + 'path': '/a/fake/path.nc', + 'lat_name': 'a lat name', + 'lon_name': 'a lon name', + 'time_name': 'a time name', + 'elevation_index': 2 + } + + self.rcmed_origin = { + 'source': 'rcmed', + 'dataset_id': 4, + 'parameter_id': 14 + } + + self.esgf_origin = { + 'source': 'esgf', + 'dataset_id': 'esgf dataset id', + 'variable': 'var' + } + + self.dap_origin = { + 'source': 'dap', + 'url': 'a fake url', + } + + self.local_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.local_origin + ) + + self.rcmed_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.rcmed_origin + ) + + self.esgf_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.esgf_origin + ) + + self.dap_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.dap_origin + ) + + self.evaluation = Evaluation( + self.local_ds, + [self.rcmed_ds, self.esgf_ds, self.dap_ds], + [] + ) + + def test_contains_only_reference_dataset(self): + new_eval = Evaluation(self.local_ds, [], []) + out = writer.generate_dataset_information(new_eval) + + self.assertTrue('reference' in out) + self.assertTrue('targets' not in out) + + def test_contains_only_target_datasets(self): + new_eval = Evaluation(None, [self.local_ds], []) + out = writer.generate_dataset_information(new_eval) + + self.assertTrue('reference' not in out) + self.assertTrue('targets' in out) + + def test_proper_reference_dataset_export(self): + out = writer.generate_dataset_information(self.evaluation) + + self.assertTrue('reference' in out) + self.assertTrue(out['reference']['data_source'] == 'local') + + def test_proper_target_datasets_export(self): + out = writer.generate_dataset_information(self.evaluation) + + self.assertTrue('targets' in out) + self.assertTrue(type(out['targets']) == type(list())) + self.assertTrue(len(out['targets']) == 3) + + class TestMetricExportGeneration(unittest.TestCase): @classmethod def setUpClass(self): @@ -450,3 +566,12 @@ class TestEvaluationSettingsGeneration(unittest.TestCase): self.assertEqual(ds_lon_max, subset[3]) self.assertEquals(str(start), subset[4]) self.assertEquals(str(end), subset[5]) + + +class FullExportTest(unittest.TestCase): + @classmethod + def setUpClass(self): + pass + + def test_full_export(self): + pass