climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From good...@apache.org
Subject [1/2] climate git commit: CLIMATE-927 - RCMES script for running multiple evaluations
Date Fri, 29 Sep 2017 06:50:41 GMT
Repository: climate
Updated Branches:
  refs/heads/master 77463b926 -> 58d6cdf16


CLIMATE-927 - RCMES script for running multiple evaluations


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fb02c749
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fb02c749
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fb02c749

Branch: refs/heads/master
Commit: fb02c74925b977abd2c03f19ea0ccd9b15faec23
Parents: 77463b9
Author: Alex <agoodm@users.noreply.github.com>
Authored: Thu Sep 28 23:11:51 2017 -0700
Committer: Alex <agoodm@users.noreply.github.com>
Committed: Thu Sep 28 23:11:51 2017 -0700

----------------------------------------------------------------------
 RCMES/CORDEX/cordex.py                      | 56 ++++++++++++++++++
 RCMES/CORDEX/metadata_extractor.py          | 35 +++++++++--
 RCMES/CORDEX/templates/CORDEX.yaml.template |  8 +--
 RCMES/metrics_and_plots.py                  |  2 +-
 RCMES/run_RCMES.py                          | 24 +++++---
 ocw/plotter.py                              | 74 ++++++++++++------------
 6 files changed, 143 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/cordex.py
----------------------------------------------------------------------
diff --git a/RCMES/CORDEX/cordex.py b/RCMES/CORDEX/cordex.py
new file mode 100644
index 0000000..dc80be4
--- /dev/null
+++ b/RCMES/CORDEX/cordex.py
@@ -0,0 +1,56 @@
+import os
+import subprocess
+import jinja2
+from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor
+
+# These should be modified. TODO: domains can also be made into separate group
+# CORDEX domain
+domain = 'NAM-44'
+
+# The output directory
+workdir = '/home/goodman/data_processing/CORDEX/analysis'
+
+# Location of osb4Mips files
+obs_dir = '/proj3/data/obs4mips'
+
+# Location of CORDEX files
+models_dir = '/proj3/data/CORDEX/{domain}/*'.format(domain=domain)
+
+# Extract metadata from model and obs files, pairing up files with the same
+# variables for separate evaluations
+obs_extractor = obs4MIPSMetadataExtractor(obs_dir)
+models_extractor = CORDEXMetadataExtractor(models_dir)
+groups = obs_extractor.group(models_extractor, 'variable')
+
+# Configuration file template, to be rendered repeatedly for each evaluation
+# run
+env =  jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),
+                          trim_blocks=True, lstrip_blocks=True)
+t = env.get_template('CORDEX.yaml.template')
+
+# Each group represents a single evaluation. Repeat the evaluation for
+# three seasons: Summer, Winter, and Annual.
+seasons = ['annual', 'winter', 'summer']
+for group in groups:
+    obs_info, models_info = group
+    instrument = obs_info['instrument']
+    variable = obs_info['variable']
+    for season in seasons:
+        configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'
+        configfile_path = os.path.join(workdir, domain, instrument,
+                                       variable, season)
+        if not os.path.exists(configfile_path):
+            os.makedirs(configfile_path)
+        configfile_path = os.path.join(configfile_path, configfile_basename)
+        with open(configfile_path, 'w') as configfile:
+            configfile.write(t.render(obs_info=obs_info, models_info=models_info,
+                                      season=season, output_dir=workdir))
+
+        # TODO: Do this in parallel. Will change this once this approach
+        # is well tested.
+        code = subprocess.call(['python', '../run_RCMES.py', configfile_path])
+        errored = []
+        if code:
+            errored.append(configfile_path)
+
+print("All runs done. The following ended with an error: {}".format(errored))

http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/metadata_extractor.py
----------------------------------------------------------------------
diff --git a/RCMES/CORDEX/metadata_extractor.py b/RCMES/CORDEX/metadata_extractor.py
index cb18946..b97d890 100644
--- a/RCMES/CORDEX/metadata_extractor.py
+++ b/RCMES/CORDEX/metadata_extractor.py
@@ -84,6 +84,13 @@ class MetadataExtractor(object):
         """
         return self.get_field('variable')
 
+    @property
+    def field_filters(self):
+        """
+        Override this to filter out specific characters contained in a field.
+        """
+        return dict()
+
     def query(self, **kwargs):
         """
         Narrow down the list of files by field names.
@@ -95,7 +102,8 @@ class MetadataExtractor(object):
         data = self.data
         for field, value in kwargs.items():
             value = value if isinstance(value, list) else [value]
-            data = [meta for meta in data if meta[field] in value]
+            data = [meta for meta in data
+                    if self._match_filter(meta, field) in value]
         return data
 
     def group(self, extractor, field):
@@ -116,7 +124,7 @@ class MetadataExtractor(object):
 
         groups = []
         for meta in results:
-            val = meta[field]
+            val = self._match_filter(meta, field)
             kwargs.update({field: val})
             match = extractor.query(**kwargs)
             groups.append((meta, match))
@@ -151,6 +159,16 @@ class MetadataExtractor(object):
         pattern = '_'.join(base[:len(self.fields)] + ['*.nc'])
         return pattern
 
+    def _match_filter(self, meta, field):
+        """
+        Filter (ignore) certain character patterns when matching a field.
+        """
+        val = meta[field]
+        if field in self.field_filters:
+            for pattern in self.field_filters[field]:
+                val = val.replace(pattern, '')
+        return val
+
     def _extract(self):
         """
         Do the actual metadata extraction from the list of filename given
@@ -183,6 +201,13 @@ class obs4MIPSMetadataExtractor(MetadataExtractor):
         fields = ['variable', 'instrument', 'processing_level', 'version']
         return fields
 
+    @property
+    def field_filters(self):
+        """
+        Field filters for CALIPSO
+        """
+        return dict(variable=['calipso', 'Lidarsr532'])
+
     def filter_filename(self, fname):
         """
         CALIPSO files have odd naming conventions, so we will use
@@ -190,8 +215,6 @@ class obs4MIPSMetadataExtractor(MetadataExtractor):
         """
         fname = os.path.basename(fname)
         fname = fname.replace('_obs4MIPs_', '_')
-        fname = fname.replace('calipso', '')
-        fname = fname.replace('Lidarsr532', '')
         return fname
 
     def get_pattern(self, fname):
@@ -202,8 +225,8 @@ class obs4MIPSMetadataExtractor(MetadataExtractor):
         offset = -2 if len(base) != 5 else -1
         pattern = '_'.join(base[:offset] + ['*.nc'])
         return pattern
-        
-        
+
+
 class CORDEXMetadataExtractor(MetadataExtractor):
     @property
     def models(self):

http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/templates/CORDEX.yaml.template
----------------------------------------------------------------------
diff --git a/RCMES/CORDEX/templates/CORDEX.yaml.template b/RCMES/CORDEX/templates/CORDEX.yaml.template
index daf6ec5..c0a1994 100644
--- a/RCMES/CORDEX/templates/CORDEX.yaml.template
+++ b/RCMES/CORDEX/templates/CORDEX.yaml.template
@@ -1,6 +1,6 @@
 {% set domain = models_info[0].domain %}
 {% set instrument = obs_info.instrument %}
-{% set variable = obs_info.variable %}
+{% set variable = models_info[0].variable %}
 {% set basename = [variable, instrument, domain, season]|join('_') %}
 workdir: {{ [output_dir, domain, instrument, variable, season]|join('/') }}
 output_netcdf_filename: {{ basename }}.nc
@@ -23,7 +23,7 @@ time:
     average_each_year: True
 
 space:
-    boundary_type: {{ domain[:3] }}
+    boundary_type: CORDEX {{ domain[:3] }}
 
 regrid:
     regrid_on_reference: True
@@ -32,12 +32,12 @@ datasets:
   - loader_name: local_split
     name: {{ instrument }}
     file_path: {{ obs_info.filename }}
-    variable_name: {{ variable }}
+    variable_name: {{ obs_info.variable }}
 {% for model_info in models_info %}
   - loader_name: local_split
     name: {{ model_info.model }}
     file_path: {{ model_info.filename }}
-    variable_name: {{ variable }}
+    variable_name: {{ model_info.variable }}
     lat_name: lat
     lon_name: lon
 {% endfor %}

http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/metrics_and_plots.py
----------------------------------------------------------------------
diff --git a/RCMES/metrics_and_plots.py b/RCMES/metrics_and_plots.py
index 78c2f14..0eb1f02 100644
--- a/RCMES/metrics_and_plots.py
+++ b/RCMES/metrics_and_plots.py
@@ -59,7 +59,7 @@ def Map_plot_bias_of_multiyear_climatology(obs_dataset, obs_name, model_datasets
 
     string_list = list(string.ascii_lowercase)
     nmodels = len(model_datasets)
-    row, column = plotter._best_grid_shape((row, column), nmodels + 1)
+    row, column = plotter._best_grid_shape(nmodels + 1, (row, column))
     ax = fig.add_subplot(row,column,1)
     if map_projection == 'npstere':
         m = Basemap(ax=ax, projection ='npstere', boundinglat=lat_min, lon_0=0,

http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/run_RCMES.py
----------------------------------------------------------------------
diff --git a/RCMES/run_RCMES.py b/RCMES/run_RCMES.py
index 7641299..f9ec2c3 100644
--- a/RCMES/run_RCMES.py
+++ b/RCMES/run_RCMES.py
@@ -25,6 +25,11 @@ from datetime import datetime
 from glob import glob
 from getpass import getpass
 import numpy as np
+
+# Need these lines to run RCMES through SSH without X11
+import matplotlib
+matplotlib.use('Agg')
+
 import ocw.utils as utils
 import ocw.dataset_processor as dsp
 from ocw.dataset import Bounds
@@ -68,7 +73,7 @@ time_info = config['time']
 temporal_resolution = time_info['temporal_resolution']
 
 # Read time info
-maximum_overlap_period = space_info.get('maximum_overlap_period', False)
+maximum_overlap_period = time_info.get('maximum_overlap_period', False)
 if not maximum_overlap_period:
     start_time = datetime.strptime(time_info['start_time'].strftime('%Y%m%d'),'%Y%m%d')
     end_time = datetime.strptime(time_info['end_time'].strftime('%Y%m%d'),'%Y%m%d')
@@ -86,7 +91,8 @@ if not 'boundary_type' in space_info:
 else:
     domain = space_info['boundary_type']
     if domain.startswith('CORDEX '):
-        domain = domain.replace('CORDEX ', '').lower()
+        domain = domain.replace('CORDEX', '').lower()
+        domain = domain.replace(' ', '')
     min_lat, max_lat, min_lon, max_lon = utils.CORDEX_boundary(domain)
 
 # Additional arguments for the DatasetLoader
@@ -110,9 +116,9 @@ multiplying_factor = np.ones(len(datasets))
 multiplying_factor[0] = fact
 names = [dataset.name for dataset in datasets]
 for i, dataset in enumerate(datasets):
-    if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
-        datasets[i] = dsp.normalize_dataset_datetimes(dataset,
-                                                      temporal_resolution)
+    res = dataset.temporal_resolution()
+    if res == 'daily' or res == 'monthly':
+        datasets[i] = dsp.normalize_dataset_datetimes(dataset, res)
         if multiplying_factor[i] != 1:
             datasets[i].values *= multiplying_factor[i]
 
@@ -148,7 +154,7 @@ else:
 for i, dataset in enumerate(datasets):
     datasets[i] = dsp.subset(dataset, bounds)
     if dataset.temporal_resolution() != temporal_resolution:
-        datasets[i] = dsp.temporal_rebin(dataset, temporal_resolution)
+        datasets[i] = dsp.temporal_rebin(datasets[i], temporal_resolution)
 
 # Temporally subset both observation and model datasets
 # for the user specified season
@@ -269,8 +275,10 @@ if nmetrics > 0:
         file_name = workdir+plot_info['file_name']
 
         print('metrics {0}/{1}: {2}'.format(imetric, nmetrics, metrics_name))
+        default_shape = (int(np.ceil(np.sqrt(ntarget + 2))),
+                         int(np.ceil(np.sqrt(ntarget + 2))))
         if metrics_name == 'Map_plot_bias_of_multiyear_climatology':
-            row, column = plot_info.get('subplots_array', (1, 1))
+            row, column = plot_info.get('subplots_array', default_shape)
             if 'map_projection' in plot_info.keys():
                 Map_plot_bias_of_multiyear_climatology(
                     reference_dataset, reference_name, target_datasets, target_names,
@@ -287,7 +295,7 @@ if nmetrics > 0:
         elif config['use_subregions']:
             if (metrics_name == 'Timeseries_plot_subregion_interannual_variability'
                 and average_each_year):
-                row, column = plot_info.get('subplots_array', (1, 1))
+                row, column = plot_info.get('subplots_array', default_shape)
                 Time_series_subregion(
                     reference_subregion_mean, reference_name, target_subregion_mean,
                     target_names, False, file_name, row, column,

http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/ocw/plotter.py
----------------------------------------------------------------------
diff --git a/ocw/plotter.py b/ocw/plotter.py
index 4896a64..7f9b092 100755
--- a/ocw/plotter.py
+++ b/ocw/plotter.py
@@ -80,12 +80,12 @@ def _nice_intervals(data, nlevs):
     else:
         mnlvl = mn
         mxlvl = mx
-    
+
     # hack to make generated intervals from mpl the same for all versions
     autolimit_mode = mpl.rcParams.get('axes.autolimit_mode')
     if autolimit_mode:
         mpl.rc('axes', autolimit_mode='round_numbers')
-    
+
     locator = mpl.ticker.MaxNLocator(nlevs)
     clevs = locator.tick_values(mnlvl, mxlvl)
     if autolimit_mode:
@@ -122,14 +122,14 @@ def _best_grid_shape(nplots, oldshape):
         # rows and columns for gridshape, automatically
         # correct it so that it fits only as many plots
         # as needed
-        while diff >= ncols:
-            nrows -= 1
+        while diff >= nrows:
+            ncols -= 1
             size = nrows * ncols
             diff = size - nplots
 
         # Don't forget to remove unnecessary columns too
-        if nrows == 1:
-            ncols = nplots
+        if ncols == 1:
+            nrows = nplots
 
         newshape = nrows, ncols
         return newshape
@@ -1096,24 +1096,24 @@ def draw_histogram(dataset_array, data_names, fname, fmt='png', nbins=10):
 
 def fill_US_states_with_color(regions, fname, fmt='png', ptitle='',
                               colors=False, values=None, region_names=None):
-    
-    ''' Fill the States over the contiguous US with colors 
-   
-    :param regions: The list of subregions(lists of US States) to be filled 
+
+    ''' Fill the States over the contiguous US with colors
+
+    :param regions: The list of subregions(lists of US States) to be filled
                     with different colors.
-    :type regions: :class:`list`  
+    :type regions: :class:`list`
     :param fname: The filename of the plot.
     :type fname: :mod:`string`
     :param fmt: (Optional) filetype for the output.
     :type fmt: :mod:`string`
     :param ptitle: (Optional) plot title.
     :type ptitle: :mod:`string`
-    :param colors: (Optional) : If True, each region will be filled  
+    :param colors: (Optional) : If True, each region will be filled
                                 with different colors without using values
-    :type colors: :class:`bool` 
-    :param values: (Optional) : If colors==False, color for each region scales 
-                                an associated element in values   
-    :type values: :class:`numpy.ndarray` 
+    :type colors: :class:`bool`
+    :param values: (Optional) : If colors==False, color for each region scales
+                                an associated element in values
+    :type values: :class:`numpy.ndarray`
     '''
 
     nregion = len(regions)
@@ -1140,7 +1140,7 @@ def fill_US_states_with_color(regions, fname, fmt='png', ptitle='',
         lons=np.empty(0)
         for shape in shapes:
             patches.append(Polygon(np.array(shape), True))
-            
+
             lons = np.append(lons, shape[:,0])
             lats = np.append(lats, shape[:,1])
         if colors:
@@ -1164,13 +1164,13 @@ def draw_plot_to_compare_trends(obs_data, ens_data, model_data,
                           fname, fmt='png', ptitle='', data_labels=None,
                           xlabel='', ylabel=''):
 
-    ''' Fill the States over the contiguous US with colors 
-   
+    ''' Fill the States over the contiguous US with colors
+
     :param obs_data: An array of observed trend and standard errors for regions
     :type obs_data: :class:'numpy.ndarray'
-    :param ens_data: An array of trend and standard errors from a multi-model ensemble for
regions 
+    :param ens_data: An array of trend and standard errors from a multi-model ensemble for
regions
     :type ens_data: : class:'numpy.ndarray'
-    :param model_data: An array of trends from models for regions 
+    :param model_data: An array of trends from models for regions
     :type model_data: : class:'numpy.ndarray'
     :param fname: The filename of the plot.
     :type fname: :mod:`string`
@@ -1182,36 +1182,36 @@ def draw_plot_to_compare_trends(obs_data, ens_data, model_data,
     :type data_labels: :mod:`list`
     :param xlabel: (Optional) a label for x-axis
     :type xlabel: :mod:`string`
-    :param ylabel: (Optional) a label for y-axis  
+    :param ylabel: (Optional) a label for y-axis
     :type ylabel: :mod:`string`
     '''
-    nregions = obs_data.shape[1] 
+    nregions = obs_data.shape[1]
 
     # Set up the figure
     fig = plt.figure()
     fig.set_size_inches((8.5, 11.))
     fig.dpi = 300
     ax = fig.add_subplot(111)
-  
-    b_plot = ax.boxplot(model_data, widths=np.repeat(0.2, nregions), positions=np.arange(nregions)+1.3)

+
+    b_plot = ax.boxplot(model_data, widths=np.repeat(0.2, nregions), positions=np.arange(nregions)+1.3)
     plt.setp(b_plot['medians'], color='black')
     plt.setp(b_plot['whiskers'], color='black')
     plt.setp(b_plot['boxes'], color='black')
     plt.setp(b_plot['fliers'], color='black')
-    ax.errorbar(np.arange(nregions)+0.8, obs_data[0,:], yerr=obs_data[1,:], 
-                fmt='o', color='r', ecolor='r')    
-    ax.errorbar(np.arange(nregions)+1., ens_data[0,:], yerr=ens_data[1,:], 
-                fmt='o', color='b', ecolor='b')    
+    ax.errorbar(np.arange(nregions)+0.8, obs_data[0,:], yerr=obs_data[1,:],
+                fmt='o', color='r', ecolor='r')
+    ax.errorbar(np.arange(nregions)+1., ens_data[0,:], yerr=ens_data[1,:],
+                fmt='o', color='b', ecolor='b')
     ax.set_xticks(np.arange(nregions)+1)
     ax.set_xlim([0, nregions+1])
-    
+
     if data_labels:
         ax.set_xticklabels(data_labels)
-    fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight') 
+    fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight')
 
-def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_names, 
-               output_file, title=None, diff_plot=False, cmap = cm.BrBG, 
-               cbar_ticks=[0.01, 0.10, 0.5, 2, 5, 25], 
+def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_names,
+               output_file, title=None, diff_plot=False, cmap = cm.BrBG,
+               cbar_ticks=[0.01, 0.10, 0.5, 2, 5, 25],
                cbar_label=['0.01', '0.10', '0.5', '2', '5', '25']):
     '''
     :param plot_data: a numpy array of data to plot (dimY, dimX)
@@ -1219,11 +1219,11 @@ def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_n
     :param plot_level: levels to plot
     :type plot_level: :class:'numpy.ndarray'
     :param x_ticks: x values where tick makrs are located
-    :type x_ticks: :class:'numpy.ndarray'                 
+    :type x_ticks: :class:'numpy.ndarray'
     :param x_names: labels for the ticks on x-axis (dimX)
     :type x_names: :class:'list'
     :param y_ticks: y values where tick makrs are located
-    :type y_ticks: :class:'numpy.ndarray'                 
+    :type y_ticks: :class:'numpy.ndarray'
     :param y_names: labels for the ticks on y-axis (dimY)
     :type y_names: :class:'list'
     :param output_file: name of output png file
@@ -1246,7 +1246,7 @@ def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_n
     dimY, dimX = plot_data.shape
     plot_data2 = np.zeros([dimY,dimX]) # sectioned array for plotting
 
-    # fontsize 
+    # fontsize
     rcParams['axes.labelsize'] = 8
     rcParams['xtick.labelsize'] = 8
     rcParams['ytick.labelsize'] = 8


Mime
View raw message