climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From huiky...@apache.org
Subject [06/12] climate git commit: debug
Date Wed, 15 Jul 2015 05:08:19 GMT
debug


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/cefd9d63
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/cefd9d63
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/cefd9d63

Branch: refs/heads/master
Commit: cefd9d632017002f1c6abb0f650e3fb8416f0eab
Parents: 69acb0e
Author: Huikyo Lee <huikyole@argo.jpl.nasa.gov>
Authored: Fri Mar 20 00:41:33 2015 -0700
Committer: Huikyo Lee <huikyole@argo.jpl.nasa.gov>
Committed: Fri Mar 20 00:41:33 2015 -0700

----------------------------------------------------------------------
 ocw/data_source/local.py | 178 ------------------------------------------
 ocw/dataset.py           |  22 +++---
 ocw/utils.py             |   4 -
 3 files changed, 11 insertions(+), 193 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/data_source/local.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py
index 3fe6497..2de03f5 100644
--- a/ocw/data_source/local.py
+++ b/ocw/data_source/local.py
@@ -26,7 +26,6 @@ import ocw.utils as utils
 import netCDF4
 import numpy
 import numpy.ma as ma
-import glob
 
 LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
 LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
@@ -212,182 +211,6 @@ def load_file(file_path,
         else:
             values = values [:,:,:,elevation_index]
 
-<<<<<<< HEAD
-    return Dataset(lats, lons, times, values, variable_name, name=name)
-
-def load_files(file_path,
-              filename_pattern,
-              variable_name,
-              elevation_index=0,
-              name='',
-              lat_name=None,
-              lon_name=None,
-              time_name=None,
-              latitude_range=None,
-              longitude_range=None):
-    ''' Load multiple NetCDF files whose file names have common patterns into a Dataset.
-    The dataset can be spatially subset.
-
-    :param file_path: Directory to the NetCDF file to load.
-    :type file_path: :mod:`string`
-
-    :param filename_pattern: Path to the NetCDF file to load.
-    :type filename_pattern: :list:`string`
-
-    :param variable_name: The variable name to load from the NetCDF file.
-    :type variable_name: :mod:`string`
-
-    :param elevation_index: (Optional) The elevation index for which data should
-        be returned. Climate data is often times 4 dimensional data. Some
-        datasets will have readins at different height/elevation levels. OCW
-        expects 3D data so a single layer needs to be stripped out when loading.
-        By default, the first elevation layer is used. If desired you may
-        specify the elevation value to use.
-    :type elevation_index: :class:`int`
-
-    :param name: (Optional) A name for the loaded dataset.
-    :type name: :mod:`string`
-
-    :param lat_name: (Optional) The latitude variable name to extract from the
-        dataset.
-    :type lat_name: :mod:`string`
-
-    :param lon_name: (Optional) The longitude variable name to extract from the
-        dataset.
-    :type lon_name: :mod:`string`
-
-    :param time_name: (Optional) The time variable name to extract from the
-        dataset.
-    :type time_name: :mod:`string`
-
-    :param latitude_range: (Optional) southern and northern boundary of the sub-region
-    :type latitude_range: :list:float   
-
-    :param longitude_range: (Optional) western and eastern boundary of the sub-region
-    :type longitude_range: :list:float   
-
-    :returns: An OCW Dataset object with the requested variable's data from
-        the NetCDF file.
-    :rtype: :class:`dataset.Dataset`
-
-    :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
-        or when the lat/lon/time variable name cannot be determined
-        automatically.
-    ''' 
-
-    netcdf_files= []
-    for pattern in filename_pattern:
-        netcdf_files.extend(glob.glob(file_path+pattern))
-    netcdf_files.sort() 
-
-    try:
-        netcdf = netCDF4.Dataset(netcdf_files[0], mode='r')
-    except RuntimeError:
-        err = "Dataset filepath is invalid. Please ensure it is correct."
-        raise ValueError(err)
-    except:
-        err = (
-            "The given file cannot be loaded. Please ensure that it is a valid "
-            "NetCDF file. If problems persist, report them to the project's "
-            "mailing list."
-        )
-        raise ValueError(err)
-
-    if not lat_name:
-        lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
-    if not lon_name:
-        lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
-    if not time_name:
-        time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
-
-    lats = netcdf.variables[lat_name][:]
-    lons = netcdf.variables[lon_name][:]
-
-    if latitude_range and longitude_range:
-        if lats.ndim == 1:
-             x_index = numpy.where((lons>=numpy.min(longitude_range)) & (lons<=numpy.max(longitude_range)))[0]
-             y_index = numpy.where((lats>=numpy.min(latitude_range)) & (lats<=numpy.max(latitude_range)))[0]

-             lats = lats[y_index]
-             lons = lons[x_index]
-        else:
-             y_index,x_index = numpy.where((lons>=numpy.min(longitude_range)) & (lons<=numpy.max(longitude_range))
& (lats>=numpy.min(latitude_range)) & (lats<=numpy.max(latitude_range)))
-             lats = lats[y_index, x_index]
-             lons = lons[y_index, x_index]
-    else:
-        y_index = np.arange(lats.shape[0]) 
-        x_index = np.arange(lons.shape[-1]) 
-
-    time_raw_values = netcdf.variables[time_name]
-    for attr, value in time_raw_values.__dict__.iteritems():
-        if 'unit' in attr.lower():
-            time_unit = value
-    times = netCDF4.num2date(time_raw_values[:], units = time_unit)
-    times = numpy.array(times)
-
-    # check the variable structure before reading data from the open file 
-    variable = netcdf.variables[variable_name]  
-    # If the values are 4D then we need to strip out the elevation index
-    if len(variable.shape) == 4:
-        # Determine the set of possible elevation dimension names excluding
-        # the list of names that are used for the lat, lon, and time values.
-        dims = netcdf.variables[variable_name].dimensions
-        dimension_names = [dim_name.encode() for dim_name in dims]
-        lat_lon_time_var_names = [lat_name, lon_name, time_name]
-
-        elev_names = set(dimension_names) - set(lat_lon_time_var_names)
-
-        # Grab the index value for the elevation values
-        level_index = dimension_names.index(elev_names.pop())
-
-        # Strip out the elevation values so we're left with a 3D array.
-        if level_index == 0:
-            values = variable[elevation_index,:,y_index,x_index]
-        elif level_index == 1:
-            values = variable[:,elevation_index,y_index,x_index]
-        else:
-            raise ValueError('The structure of this variable does not follow the community
standard')
-        if len(netcdf_files) >1:
-            for netcdf_file in netcdf_files[1:]:
-                netcdf.close()
-                netcdf = netCDF4.Dataset(netcdf_file, mode='r')
-                time_raw_values = netcdf.variables[time_name]
-                for attr, value in time_raw_values.__dict__.iteritems():
-                    if 'unit' in attr.lower():
-                        time_unit = value
-                times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units =
time_unit))
-                if level_index == 0:
-                    values = numpy.concatenate((values, netcdf.variables[variable_name][elevation_index,:,y_index,x_index]),
axis=0)
-                elif level_index == 1:
-                    values = numpy.concatenate((values, netcdf.variables[variable_name][:,elevation_index,y_index,x_index]),
axis=0)
-
-    elif len(variable.shape) == 3:
-        values = variable[:,y_index,x_index]
-        
-        if len(netcdf_files) >1:
-            for netcdf_file in netcdf_files[1:]:
-                netcdf.close()
-                netcdf = netCDF4.Dataset(netcdf_file, mode='r')
-                time_raw_values = netcdf.variables[time_name]
-                for attr, value in time_raw_values.__dict__.iteritems():
-                    if 'unit' in attr.lower():
-                        time_unit = value
-                times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units=time_unit))
-                values = numpy.concatenate((values, netcdf.variables[variable_name][:,y_index,x_index]),
axis=0)
-    elif len(variable.shape) == 2:
-        values = (variable[y_index,x_index]).reshape((1,y_index.size,x_index.size))
-        if len(netcdf_files) >1:
-            for netcdf_file in netcdf_files[1:]:
-                netcdf.close()
-                netcdf = netCDF4.Dataset(netcdf_file, mode='r')
-                time_raw_values = netcdf.variables[time_name]
-                for attr, value in time_raw_values.__dict__.iteritems():
-                    if 'unit' in attr.lower():
-                        time_unit = value
-                times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units=time_unit))
-                values = numpy.concatenate((values, (netcdf.variables[variable_name][y_index,x_index]).reshape((1,y_index.size,x_index.size))),
axis=0)
-    return Dataset(lats, lons, times, values, variable_name, name=name)
-
-=======
     origin = {
         'source': 'local',
         'path': file_path,
@@ -399,4 +222,3 @@ def load_files(file_path,
 
     return Dataset(lats, lons, times, values, variable=variable_name,
                    units=variable_unit, name=name, origin=origin)
->>>>>>> upstream/master

http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/dataset.py
----------------------------------------------------------------------
diff --git a/ocw/dataset.py b/ocw/dataset.py
index 48e7a22..ee86532 100644
--- a/ocw/dataset.py
+++ b/ocw/dataset.py
@@ -69,7 +69,7 @@ class Dataset:
         :raises: ValueError
         '''
         self._validate_inputs(lats, lons, times, values)
-        #lats, lons, values = utils.normalize_lat_lon_values(lats, lons, values)
+        lats, lons, values = utils.normalize_lat_lon_values(lats, lons, values)
 
         self.lats = lats
         self.lons = lons
@@ -180,17 +180,17 @@ class Dataset:
             err_msg = "Longitude Array should be 1 dimensional. %s dimensions found." % lon_dim
         elif time_dim != 1:
             err_msg = "Time Array should be 1 dimensional.  %s dimensions found." % time_dim
-        #elif value_dim != 3:
-        #    err_msg = "Value Array should be 3 dimensional.  %s dimensions found." % value_dim
+        elif value_dim != 3:
+            err_msg = "Value Array should be 3 dimensional.  %s dimensions found." % value_dim
         # Finally check that the Values array conforms to the proper shape
-        #elif values.shape != (time_count, lat_count, lon_count):
-        #    err_msg = """Value Array must be of shape (times, lats, lons).
-#Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count,
-#                                                            lat_count,
-#                                                            lon_count,
-#                                                            values.shape[0],
-#                                                            values.shape[1],
-#                                                            values.shape[2])
+        elif values.shape != (time_count, lat_count, lon_count):
+            err_msg = """Value Array must be of shape (times, lats, lons).
+Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count,
+                                                            lat_count,
+                                                            lon_count,
+                                                            values.shape[0],
+                                                            values.shape[1],
+                                                            values.shape[2])
         if err_msg:
             logger.error(err_msg)
             raise ValueError(err_msg)

http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/utils.py
----------------------------------------------------------------------
diff --git a/ocw/utils.py b/ocw/utils.py
index e3d3311..e778d40 100755
--- a/ocw/utils.py
+++ b/ocw/utils.py
@@ -289,7 +289,6 @@ def calc_climatology_year(dataset):
 
     return annually_mean, total_mean
 
-<<<<<<< HEAD
 def calc_climatology_season(month_start, month_end, dataset):
     ''' Calculate seasonal mean and time series for given months.
 
@@ -345,6 +344,3 @@ def calc_climatology_monthly(dataset):
         raise ValueError(error)
     else:
         return reshape_monthly_to_annually(dataset).mean(axis=0)
-
-=======
->>>>>>> cfb120e5eaea004e850b884e49d41f8a0c269b75


Mime
View raw message