climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jo...@apache.org
Subject [1/2] climate git commit: CLIMATE-633 - Add support for loading WRF data 2D formatted files
Date Mon, 11 May 2015 15:15:53 GMT
Repository: climate
Updated Branches:
  refs/heads/master 89a96c0fe -> 6ab02d8f6


CLIMATE-633 - Add support for loading WRF data 2D formatted files

- Add load_WRF_2d_files to datasource.local for handling the Weather
  Research and Forecasting 2D field datasets.
- Update normalize_lat_lon_values to handle multi-dim lat/lon fields.


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/13692480
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/13692480
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/13692480

Branch: refs/heads/master
Commit: 136924803f05841e96cad894eb8f073fecee648a
Parents: 89a96c0
Author: Huikyo Lee <hlee21@dali14.prv.cube>
Authored: Tue Apr 28 15:39:46 2015 -0400
Committer: Michael Joyce <joyce@apache.org>
Committed: Mon May 11 08:13:51 2015 -0700

----------------------------------------------------------------------
 ocw/data_source/local.py | 46 ++++++++++++++++++++++++++
 ocw/dataset.py           |  9 +++---
 ocw/utils.py             | 75 ++++++++++++++++++++++---------------------
 3 files changed, 89 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/13692480/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index 2de03f5..474dffb 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -17,6 +17,8 @@
 
 import calendar
 from datetime import timedelta ,datetime
+from time import strptime
+from glob import glob
 import re
 import string
 
@@ -109,6 +111,50 @@ def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
     )
     raise ValueError(error)
 
+def load_WRF_2d_files(file_path,
+                      filename_pattern,
+                      variable_name,
+                      name=''):
+    ''' Load multiple WRF (or nuWRF) original output files containing 2D fields such as precipitation
and surface variables into a Dataset.
+    The dataset can be spatially subset.
+    :param file_path: Directory to the NetCDF file to load.
+    :type file_path: :mod:`string`
+    :param filename_pattern: Path to the NetCDF file to load.
+    :type filename_pattern: :list:`string`
+    :param variable_name: The variable name to load from the NetCDF file.
+    :type variable_name: :mod:`string`
+    :param name: (Optional) A name for the loaded dataset.
+    :type name: :mod:`string`
+    :returns: An OCW Dataset object with the requested variable's data from
+        the NetCDF file.
+    :rtype: :class:`dataset.Dataset`
+    :raises ValueError: 
+    '''                  
+    
+    WRF_files = []
+    for pattern in filename_pattern:
+        WRF_files.extend(glob(file_path + pattern))
+    WRF_files.sort()
+  
+    file_object_first = netCDF4.Dataset(WRF_files[0])
+    lats = file_object_first.variables['XLAT'][0,:]
+    lons = file_object_first.variables['XLONG'][0,:]
+
+    times = []
+    for ifile, file in enumerate(WRF_files):
+        file_object = netCDF4.Dataset(file)
+        time_struct_parsed = strptime(file[-19:],"%Y-%m-%d_%H:%M:%S")     
+        for ihour in numpy.arange(24):
+            times.append(datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour))
+        values0= file_object.variables[variable_name][:]
+        if ifile == 0:
+            values = file_object.variables[variable_name][:]
+        else:
+            values = numpy.concatenate((values, file_object.variables[variable_name][:]))

+        file_object.close()
+    times = numpy.array(times)
+    return Dataset(lats, lons, times, values, variable_name, name=name)
+
 def load_file(file_path,
               variable_name,
               variable_unit = None,

http://git-wip-us.apache.org/repos/asf/climate/blob/13692480/ocw/dataset.py
----------------------------------------------------------------------
diff --git a/ocw/dataset.py b/ocw/dataset.py
index d231f0e..d33bdd4 100644
--- a/ocw/dataset.py
+++ b/ocw/dataset.py
@@ -172,13 +172,12 @@ class Dataset:
         value_dim = len(values.shape)
         lat_count = lats.shape[0]
         lon_count = lons.shape[0]
+        
+        if lat_dim == 2 and lon_dim == 2:
+            lon_count = lons.shape[1]
         time_count = times.shape[0]
         
-        if lat_dim != 1:
-            err_msg = "Latitude Array should be 1 dimensional.  %s dimensions found." % lat_dim
-        elif lon_dim != 1:
-            err_msg = "Longitude Array should be 1 dimensional. %s dimensions found." % lon_dim
-        elif time_dim != 1:
+        if time_dim != 1:
             err_msg = "Time Array should be 1 dimensional.  %s dimensions found." % time_dim
         elif value_dim < 2:
             err_msg = "Value Array should be at least 2 dimensional.  %s dimensions found."
% value_dim

http://git-wip-us.apache.org/repos/asf/climate/blob/13692480/ocw/utils.py
----------------------------------------------------------------------
diff --git a/ocw/utils.py b/ocw/utils.py
index f367dab..e8a004c 100644
--- a/ocw/utils.py
+++ b/ocw/utils.py
@@ -179,42 +179,45 @@ def normalize_lat_lon_values(lats, lons, values):
 
     :raises ValueError: If the lat/lon values are not sorted.
     '''
-    # Avoid unnecessary shifting if all lons are higher than 180
-    if lons.min() > 180:
-        lons -= 360
-
-    # Make sure lats and lons are monotonically increasing
-    lats_decreasing = np.diff(lats) < 0
-    lons_decreasing = np.diff(lons) < 0
-
-    # If all values are decreasing then they just need to be reversed
-    lats_reversed, lons_reversed = lats_decreasing.all(), lons_decreasing.all()
-
-    # If the lat values are unsorted then raise an exception
-    if not lats_reversed and lats_decreasing.any():
-        raise ValueError('Latitudes must be sorted.')
-
-    # Perform same checks now for lons
-    if not lons_reversed and lons_decreasing.any():
-        raise ValueError('Longitudes must be sorted.')
-
-    # Also check if lons go from [0, 360), and convert to [-180, 180)
-    # if necessary
-    lons_shifted = lons.max() > 180
-    lats_out, lons_out, data_out = lats[:], lons[:], values[:]
-    # Now correct data if latlon grid needs to be shifted
-    if lats_reversed:
-        lats_out = lats_out[::-1]
-        data_out = data_out[..., ::-1, :]
-
-    if lons_reversed:
-        lons_out = lons_out[::-1]
-        data_out = data_out[..., ::-1]
-
-    if lons_shifted:
-        data_out, lons_out = shiftgrid(180, data_out, lons_out, start=False)
-
-    return lats_out, lons_out, data_out
+    if lats.ndim ==1 and lons.ndim ==1:
+        # Avoid unnecessary shifting if all lons are higher than 180
+        if lons.min() > 180:
+            lons -= 360
+
+    	# Make sure lats and lons are monotonically increasing
+    	lats_decreasing = np.diff(lats) < 0
+    	lons_decreasing = np.diff(lons) < 0
+
+    	# If all values are decreasing then they just need to be reversed
+    	lats_reversed, lons_reversed = lats_decreasing.all(), lons_decreasing.all()
+
+    	# If the lat values are unsorted then raise an exception
+    	if not lats_reversed and lats_decreasing.any():
+            raise ValueError('Latitudes must be sorted.')
+
+    	# Perform same checks now for lons
+    	if not lons_reversed and lons_decreasing.any():
+            raise ValueError('Longitudes must be sorted.')
+
+    	# Also check if lons go from [0, 360), and convert to [-180, 180)
+    	# if necessary
+    	lons_shifted = lons.max() > 180
+    	lats_out, lons_out, data_out = lats[:], lons[:], values[:]
+    	# Now correct data if latlon grid needs to be shifted
+    	if lats_reversed:
+            lats_out = lats_out[::-1]
+            data_out = data_out[..., ::-1, :]
+
+    	if lons_reversed:
+            lons_out = lons_out[::-1]
+            data_out = data_out[..., ::-1]
+
+    	if lons_shifted:
+            data_out, lons_out = shiftgrid(180, data_out, lons_out, start=False)
+
+        return lats_out, lons_out, data_out
+    else:
+        return lats, lons, values
 
 
 def reshape_monthly_to_annually(dataset):


Mime
View raw message