climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From good...@apache.org
Subject [1/5] climate git commit: CLIMATE-883 - Ensure python 3 builds pass unit tests
Date Mon, 24 Oct 2016 23:27:10 GMT
Repository: climate
Updated Branches:
  refs/heads/master 088706d82 -> 386f7b924


http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_dataset.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_dataset.py b/ocw/tests/test_dataset.py
index a51cddc..076c217 100644
--- a/ocw/tests/test_dataset.py
+++ b/ocw/tests/test_dataset.py
@@ -43,13 +43,13 @@ class TestDatasetAttributes(unittest.TestCase):
                                     origin=self.origin)
 
     def test_lats(self):
-        self.assertItemsEqual(self.test_dataset.lats, self.lat)
+        np.testing.assert_array_equal(self.test_dataset.lats, self.lat)
 
     def test_lons(self):
-        self.assertItemsEqual(self.test_dataset.lons, self.lon)
+        np.testing.assert_array_equal(self.test_dataset.lons, self.lon)
 
     def test_times(self):
-        self.assertItemsEqual(self.test_dataset.times, self.time)
+        np.testing.assert_array_equal(self.test_dataset.times, self.time)
 
     def test_values(self):
         self.assertEqual(self.test_dataset.values.all(), self.value.all())

http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_dataset_processor.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_dataset_processor.py b/ocw/tests/test_dataset_processor.py
index 9538e91..9beee75 100644
--- a/ocw/tests/test_dataset_processor.py
+++ b/ocw/tests/test_dataset_processor.py
@@ -99,16 +99,16 @@ class TestTemporalRebinWithTimeIndex(unittest.TestCase):
 
     def test_time_dimension_multiple_of_orig_time_dimension(self):
         # ten_year_dataset.times.size is 120
-        nt_avg = self.ten_year_dataset.times.size / 2
+        nt_avg = self.ten_year_dataset.times.size // 2
         # Temporal Rebin to exactly 2 (time) values
         dataset = dp.temporal_rebin_with_time_index(
             self.ten_year_dataset, nt_avg)
         start_time = self.ten_year_dataset.times[0]
         # First month of the middle year
-        middle_element = self.ten_year_dataset.times.size / 2
+        middle_element = self.ten_year_dataset.times.size // 2
         end_time = self.ten_year_dataset.times[middle_element]
         self.assertEqual(dataset.times.size,
-                         self.ten_year_dataset.times.size / nt_avg)
+                         self.ten_year_dataset.times.size // nt_avg)
         np.testing.assert_array_equal(dataset.times, [start_time, end_time])
 
     def test_time_dimension_not_multiple_of_orig_time_dimension(self):
@@ -119,7 +119,7 @@ class TestTemporalRebinWithTimeIndex(unittest.TestCase):
             self.ten_year_dataset, nt_avg)
         new_times = self.ten_year_dataset.times[::11][:-1]
         self.assertEqual(dataset.times.size,
-                         self.ten_year_dataset.times.size / nt_avg)
+                         self.ten_year_dataset.times.size // nt_avg)
         np.testing.assert_array_equal(dataset.times, new_times)
 
     def test_returned_dataset_attributes(self):

http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_local.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_local.py b/ocw/tests/test_local.py
index 645f486..bff2dd1 100644
--- a/ocw/tests/test_local.py
+++ b/ocw/tests/test_local.py
@@ -15,12 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Needed Python 2/3 urllib compatability
+try:
+    from urllib.request import urlretrieve
+except ImportError:
+    from urllib import urlretrieve
+    
 import datetime
 import unittest
 import os
-import urllib
-import numpy
 import netCDF4
+import numpy as np
 
 import ocw.data_source.local as local
 
@@ -51,42 +56,42 @@ class test_load_file(unittest.TestCase):
 
     def test_function_load_file_lats(self):
         """To test load_file function for latitudes"""
-        self.assertItemsEqual(local.load_file(
+        np.testing.assert_array_equal(local.load_file(
             self.file_path, "value").lats, self.latitudes)
 
     def test_function_load_file_lons(self):
         """To test load_file function for longitudes"""
-        self.assertItemsEqual(local.load_file(
+        np.testing.assert_array_equal(local.load_file(
             self.file_path, "value").lons, self.longitudes)
 
     def test_function_load_file_times(self):
         """To test load_file function for times"""
         newTimes = datetime.datetime(2001, 1, 1), datetime.datetime(
             2001, 2, 1), datetime.datetime(2001, 3, 1)
-        self.assertItemsEqual(local.load_file(
+        np.testing.assert_array_equal(local.load_file(
             self.file_path, "value").times, newTimes)
 
     def test_function_load_file_alt_lats(self):
         """To test load_file function for lats with different variable names"""
-        self.assertItemsEqual(local.load_file(
+        np.testing.assert_array_equal(local.load_file(
             self.file_path, "value", lat_name="alt_lat").lats, self.alt_lats)
 
     def test_function_load_file_alt_lons(self):
         """To test load_file function for lons with different variable names"""
-        self.assertItemsEqual(local.load_file(
+        np.testing.assert_array_equal(local.load_file(
             self.file_path, "value", lon_name="alt_lon").lons, self.alt_lons)
 
     def test_function_load_file_alt_times(self):
         """To test load_file function for times with different variable names"""
         newTimes = datetime.datetime(2001, 4, 1), datetime.datetime(
             2001, 5, 1), datetime.datetime(2001, 6, 1)
-        self.assertItemsEqual(local.load_file(
+        np.testing.assert_array_equal(local.load_file(
             self.file_path, "value", time_name="alt_time").times, newTimes)
 
     def test_function_load_file_values(self):
         """To test load_file function for values"""
         new_values = self.values[:, 0, :, :]
-        self.assertTrue(numpy.allclose(local.load_file(
+        self.assertTrue(np.allclose(local.load_file(
             self.file_path, "value").values, new_values))
 
     def test_custom_dataset_name(self):
@@ -125,7 +130,7 @@ class TestLoadMultipleFiles(unittest.TestCase):
     def test_function_load_multiple_files_lons(self):
         """To test load_multiple_file function for longitudes"""
         dataset = local.load_multiple_files(self.file_path, "value")
-        self.assertItemsEqual(dataset[0].lons, self.longitudes)
+        np.testing.assert_array_equal(dataset[0].lons, self.longitudes)
 
     def test_function_load_multiple_files_times(self):
         """To test load_multiple_files function for times"""
@@ -133,14 +138,14 @@ class TestLoadMultipleFiles(unittest.TestCase):
 
         newTimes = datetime.datetime(2001, 1, 1), datetime.datetime(
             2001, 2, 1), datetime.datetime(2001, 3, 1)
-        self.assertItemsEqual(dataset[0].times, newTimes)
+        np.testing.assert_array_equal(dataset[0].times, newTimes)
 
     def test_function_load_multiple_files_values(self):
         """To test load_multiple_files function for values"""
         new_values = self.values[:, 0, :, :]
         dataset = local.load_multiple_files(
             self.file_path, "value")
-        self.assertTrue(numpy.allclose(dataset[0].values, new_values))
+        self.assertTrue(np.allclose(dataset[0].values, new_values))
 
     def test_load_multiple_files_custom_dataset_name(self):
         """Test adding a custom name to a dataset"""
@@ -192,40 +197,40 @@ class TestLoadDatasetFromMultipleNetcdfFiles(unittest.TestCase):
 
     def test_function_load_dataset_from_multiple_netcdf_files_lats(self):
         """To test load_multiple_files function for times"""
-        _, self.latitudes = numpy.meshgrid(self.longitudes, self.latitudes)
-        numpy.testing.assert_array_equal(self.dataset.lats, self.latitudes)
+        _, self.latitudes = np.meshgrid(self.longitudes, self.latitudes)
+        np.testing.assert_array_equal(self.dataset.lats, self.latitudes)
 
     def test_function_load_dataset_from_multiple_netcdf_files_lons(self):
         """To test load_multiple_files function for times"""
-        self.longitudes, _ = numpy.meshgrid(self.longitudes, self.latitudes)
-        numpy.testing.assert_array_equal(self.dataset.lons, self.longitudes)
+        self.longitudes, _ = np.meshgrid(self.longitudes, self.latitudes)
+        np.testing.assert_array_equal(self.dataset.lons, self.longitudes)
 
     def test_function_load_dataset_from_multiple_netcdf_files_times(self):
         """To test load_multiple_files function for times"""
         newTimes = datetime.datetime(2001, 1, 1), datetime.datetime(
             2001, 2, 1), datetime.datetime(2001, 3, 1)
-        self.assertItemsEqual(self.dataset.times, newTimes)
+        np.testing.assert_array_equal(self.dataset.times, newTimes)
 
     def test_function_load_dataset_from_multiple_netcdf_files_alt_lats(self):
         """To test load_multiple_files function for non-default lats"""
-        _, self.alt_lats = numpy.meshgrid(self.alt_lons, self.alt_lats)
-        numpy.testing.assert_array_equal(self.alt_dataset.lats, self.alt_lats)
+        _, self.alt_lats = np.meshgrid(self.alt_lons, self.alt_lats)
+        np.testing.assert_array_equal(self.alt_dataset.lats, self.alt_lats)
 
     def test_function_load_dataset_from_multiple_netcdf_files_alt_lons(self):
         """To test load_multiple_files function for non-default lons"""
-        self.alt_lons, _ = numpy.meshgrid(self.alt_lons, self.alt_lats)
-        numpy.testing.assert_array_equal(self.alt_dataset.lons, self.alt_lons)
+        self.alt_lons, _ = np.meshgrid(self.alt_lons, self.alt_lats)
+        np.testing.assert_array_equal(self.alt_dataset.lons, self.alt_lons)
 
     def test_function_load_dataset_from_multiple_netcdf_files_alt_times(self):
         """To test load_multiple_files function for non-default times"""
         newTimes = datetime.datetime(2001, 4, 1), datetime.datetime(
             2001, 5, 1), datetime.datetime(2001, 6, 1)
-        self.assertItemsEqual(self.alt_dataset.times, newTimes)
+        np.testing.assert_array_equal(self.alt_dataset.times, newTimes)
 
     def test_function_load_dataset_from_multiple_netcdf_files_values(self):
         """To test load_multiple_files function for values"""
         new_values = self.values[:, 0, :, :]
-        self.assertTrue(numpy.allclose(self.dataset.values, new_values))
+        self.assertTrue(np.allclose(self.dataset.values, new_values))
 
 
 class test_get_netcdf_variable_names(unittest.TestCase):
@@ -233,7 +238,7 @@ class test_get_netcdf_variable_names(unittest.TestCase):
     test_model = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc"
 
     def setUp(self):
-        urllib.urlretrieve(self.file_path + self.test_model, self.test_model)
+        urlretrieve(self.file_path + self.test_model, self.test_model)
         self.invalid_netcdf_path = create_invalid_dimensions_netcdf_object()
         self.netcdf = netCDF4.Dataset(self.test_model, mode='r')
 
@@ -295,14 +300,14 @@ def create_netcdf_object():
                                         )
 
     # To latitudes and longitudes for five values
-    latitudes_data = numpy.arange(5.)
-    longitudes_data = numpy.arange(150., 155.)
+    latitudes_data = np.arange(5.)
+    longitudes_data = np.arange(150., 155.)
     # Three months of data.
-    times_data = numpy.arange(3)
+    times_data = np.arange(3)
     # Two levels
     levels_data = [100, 200]
     # Create 150 values
-    values_data = numpy.array([i for i in range(150)])
+    values_data = np.array([i for i in range(150)])
     # Reshape values to 4D array (level, time, lats, lons)
     values_data = values_data.reshape(len(times_data), len(
         levels_data), len(latitudes_data), len(longitudes_data))
@@ -347,23 +352,23 @@ def create_invalid_dimensions_netcdf_object():
                                          'lon_dim')
                                         )
     # To latitudes and longitudes for five values
-    latitudes = range(0, 5)
-    longitudes = range(200, 205)
+    flatitudes = list(range(0, 5))
+    flongitudes = list(range(200, 205))
     # Three months of data
-    times = range(3)
+    ftimes = list(range(3))
     # Two levels
-    levels = [100, 200]
+    flevels = [100, 200]
     # Create 150 values
-    values = numpy.array([i for i in range(150)])
+    fvalues = np.array([i for i in range(150)])
     # Reshape values to 4D array (level, time, lats, lons)
-    values = values.reshape(len(levels), len(
-        times), len(latitudes), len(longitudes))
+    fvalues = fvalues.reshape(len(flevels), len(
+        times), len(flatitudes), len(flongitudes))
     # Ingest values to netCDF file
-    latitudes[:] = latitudes
-    longitudes[:] = longitudes
-    times[:] = times
-    levels[:] = levels
-    values[:] = values
+    latitudes[:] = flatitudes
+    longitudes[:] = flongitudes
+    times[:] = ftimes
+    levels[:] = flevels
+    values[:] = fvalues
     # Assign time info to time variable
     netCDF_file.variables['time'].units = 'months since 2001-01-01 00:00:00'
     netCDF_file.close()

http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_plotter.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_plotter.py b/ocw/tests/test_plotter.py
index 9020fd2..19ba66a 100644
--- a/ocw/tests/test_plotter.py
+++ b/ocw/tests/test_plotter.py
@@ -98,7 +98,7 @@ class TestFigshapeFunction(unittest.TestCase):
         gridshape = (2, 2)
         expected_width = 5.5
         expected_height = 5.5
-        width, height = plotter._fig_size(gridshape, aspect=(4 / 3))
+        width, height = plotter._fig_size(gridshape, aspect=(4 // 3))
         self.assertEqual(width, expected_width)
         self.assertEqual(height, expected_height)
 
@@ -106,7 +106,7 @@ class TestFigshapeFunction(unittest.TestCase):
         gridshape = (567, 1223)
         expected_width = 11.0
         expected_height = 5.5
-        width, height = plotter._fig_size(gridshape, aspect=(16 / 9))
+        width, height = plotter._fig_size(gridshape, aspect=(16 // 9))
         self.assertEqual(width, expected_width)
         self.assertEqual(height, expected_height)
 

http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_rcmed.py
----------------------------------------------------------------------
diff --git a/ocw/tests/test_rcmed.py b/ocw/tests/test_rcmed.py
index 35975a1..554776b 100644
--- a/ocw/tests/test_rcmed.py
+++ b/ocw/tests/test_rcmed.py
@@ -17,11 +17,9 @@
 
 import unittest
 import datetime
-import numpy
+import numpy as np
 import pickle
-import inspect
 import os
-import test_rcmed  # Import test_rcmed so we can use inspect to get the path
 import ocw.data_source.rcmed as rcmed
 
 
@@ -47,33 +45,31 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
         # of start_time and end_time
         self.start_time_for_url = "20020801T0000Z"
         self.end_time_for_url = "20021001T2359Z"
-        self.url = "http://rcmes.jpl.nasa.gov/query-api/query.php?"
-        self.lats = numpy.arange(50.5, 70, 1)
-        self.lons = numpy.arange(1.5, 15, 1)
+        self.url = b"http://rcmes.jpl.nasa.gov/query-api/query.php?"
+        self.lats = np.arange(50.5, 70, 1)
+        self.lons = np.arange(1.5, 15, 1)
         # In this parameter, two days of 10/20 and 10/21 have been missed.
         self.times_list = [datetime.datetime(
             2002, 8, 31) + datetime.timedelta(days=x) for x in range(0, 62)]
         self.times_list.remove(datetime.datetime(2002, 10, 20))
         self.times_list.remove(datetime.datetime(2002, 10, 21))
-        self.times = numpy.array(self.times_list)
+        self.times = np.array(self.times_list)
 
         # Get the full path to test_rcmed
-        self.file_path = os.path.abspath(inspect.getfile(test_rcmed))
-        # Remove the end of our path so we're left with the directory in which
-        # test_rcmed.py is stored.
-        self.file_path = self.file_path.split('test_rcmed')[0]
+        self.file_path = os.path.dirname(__file__)
 
         # Grab the parameter values file
-        params_file = open(os.path.join(
-            self.file_path, "parameters_values.p"), "rb")
-        self.values = pickle.load(params_file)
+        self.values = np.ma.array(np.load(os.path.join(
+            self.file_path, "parameter_values.npy")))
+
         # Grab the parameter metadata file
         meta_file = open(os.path.join(
             self.file_path, "parameters_metadata_output.p"), "rb")
         self.param_metadata_output = pickle.load(meta_file)
 
     def return_text(self, url):
-        if url == self.url + ("datasetId={0}&"
+        url = _force_bytes(url)
+        query = _force_bytes(("datasetId={0}&"
                               "parameterId={1}&"
                               "latMin={2}&"
                               "latMax={3}&"
@@ -87,17 +83,18 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
                                                     self.min_lon,
                                                     self.max_lon,
                                                     self.start_time_for_url,
-                                                    self.end_time_for_url):
+                                                    self.end_time_for_url))
+        if url == self.url + query:
             return open(os.path.join(self.file_path,
-                                     "parameter_dataset_text.txt"))
-        elif url == self.url + "&param_info=yes":
+                                     "parameter_dataset_text.txt"), 'rb')
+        elif url == self.url + b"&param_info=yes":
             return open(os.path.join(self.file_path,
-                                     "parameters_metadata_text.txt"))
+                                     "parameters_metadata_text.txt"), 'rb')
         else:
             raise Exception("The URL is not acceptable.")
 
     def test_function_get_parameters_metadata(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         self.assertEqual(rcmed.get_parameters_metadata(),
                          self.param_metadata_output)
 
@@ -105,7 +102,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
         self.dataset_id = 6
         self.parameter_id = 32
         self.end_time_for_url = "20021031T0000Z"
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id,
                                                          self.parameter_id,
                                                          self.min_lat,
@@ -117,7 +114,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
                                  self.lats)
 
     def test_function_parameter_dataset_lats(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id,
                                                          self.parameter_id,
                                                          self.min_lat,
@@ -129,7 +126,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
                                  self.lats)
 
     def test_function_parameter_dataset_lons(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id,
                                                          self.parameter_id,
                                                          self.min_lat,
@@ -141,7 +138,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
                                  self.lons)
 
     def test_function_parameter_dataset_times(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id,
                                                          self.parameter_id,
                                                          self.min_lat,
@@ -153,7 +150,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
                                  self.times)
 
     def test_function_parameter_dataset_values(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id,
                                                          self.parameter_id,
                                                          self.min_lat,
@@ -166,7 +163,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
                                  self.values.flatten())
 
     def test_function_parameter_dataset_name(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         ds = rcmed.parameter_dataset(self.dataset_id,
                                      self.parameter_id,
                                      self.min_lat,
@@ -179,7 +176,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
         self.assertEquals(ds.name, 'foo')
 
     def test_dataset_origin(self):
-        rcmed.urllib2.urlopen = self.return_text
+        rcmed.urlopen = self.return_text
         ds = rcmed.parameter_dataset(self.dataset_id,
                                      self.parameter_id,
                                      self.min_lat,
@@ -194,5 +191,12 @@ class test_rcmed(unittest.TestCase, CustomAssertions):
         self.assertEquals(ds.origin['dataset_id'], self.dataset_id)
         self.assertEquals(ds.origin['parameter_id'], self.parameter_id)
 
+
+def _force_bytes(s, encoding='utf-8'):
+    if hasattr(s, 'encode'):
+        s = s.encode(encoding=encoding)
+    
+    return s
+
 if __name__ == '__main__':
     unittest.main()

http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/utils.py
----------------------------------------------------------------------
diff --git a/ocw/utils.py b/ocw/utils.py
index 4876966..468ef58 100755
--- a/ocw/utils.py
+++ b/ocw/utils.py
@@ -364,7 +364,7 @@ def calc_time_series(dataset):
     '''
 
     t_series = []
-    for t in xrange(dataset.values.shape[0]):
+    for t in range(dataset.values.shape[0]):
         t_series.append(dataset.values[t, :, :].mean())
 
     return t_series
@@ -468,15 +468,25 @@ def shapefile_boundary(boundary_type, region_names):
     shapefile_dir = os.sep.join([os.path.dirname(__file__), 'shape'])
     map_read.readshapefile(os.path.join(shapefile_dir, boundary_type),
                            boundary_type)
+                           
+    # Note: The shapefile may contain countries or states with latin letters.
+    # Hence, the output of readshapefile can be a mix of ascii and unicode
+    # strings. Because python 2 and 3 treat them differently, we must
+    # explicitly check.
     if boundary_type == 'us_states':
         for region_name in region_names:
+            region_name = _force_unicode(region_name)
             for iregion, region_info in enumerate(map_read.us_states_info):
-                if region_info['st'] == region_name:
+                state = _force_unicode(region_info['st'], 'latin-1')
+                if state == region_name:
                     regions.append(np.array(map_read.us_states[iregion]))
     elif boundary_type == 'countries':
         for region_name in region_names:
+            region_name = _force_unicode(region_name)
             for iregion, region_info in enumerate(map_read.countries_info):
-                if region_info['COUNTRY'].replace(" ", "").lower() == region_name.replace("
", "").lower():
+                country = _force_unicode(region_info['COUNTRY'], 'latin-1')
+                if (country.replace(" ", "").lower() ==
+                    region_name.replace(" ", "").lower()):
                     regions.append(np.array(map_read.countries[iregion]))
     return regions
 
@@ -595,3 +605,12 @@ def convert_lat_lon_2d_array(lon, lat):
         return np.meshgrid(lon, lat)
     if lon.ndim == 2 and lat.ndim == 2:
         return lon, lat
+
+def _force_unicode(s, encoding='utf-8'):
+    '''
+    If the input is bytes, convert to unicode, otherwise return the input
+    '''
+    if hasattr(s, 'decode'):
+        s = s.decode(encoding=encoding)
+        
+    return s


Mime
View raw message