climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From good...@apache.org
Subject climate git commit: Fixed issues with ESGF loader
Date Tue, 09 Aug 2016 23:20:06 GMT
Repository: climate
Updated Branches:
  refs/heads/master ddae7589c -> 625eec2d2


Fixed issues with ESGF loader


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/625eec2d
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/625eec2d
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/625eec2d

Branch: refs/heads/master
Commit: 625eec2d2454d699e9a0dfe4e9d9a73f6c55ed57
Parents: ddae758
Author: Alex Goodman <agoodm@users.noreply.github.com>
Authored: Tue Aug 9 15:56:28 2016 -0700
Committer: Alex Goodman <agoodm@users.noreply.github.com>
Committed: Tue Aug 9 15:56:28 2016 -0700

----------------------------------------------------------------------
 .../ESGF_example/ESGF-Example.yaml              |  4 +-
 RCMES/run_RCMES.py                              | 46 +++++++++++---------
 ocw/data_source/esgf.py                         | 10 ++---
 ocw/esgf/logon.py                               |  6 +--
 4 files changed, 33 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml
----------------------------------------------------------------------
diff --git a/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml b/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml
index eb65a3d..29d06c8 100644
--- a/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml
+++ b/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml
@@ -39,13 +39,13 @@ regrid:
 
 datasets:
     reference:
-        - loader_name: ESGF
+        - loader_name: esgf
           name: AVISO
           dataset_id: obs4MIPs.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov
           variable_name: zos
 
     targets:
-        - loader_name: ESGF
+        - loader_name: esgf
           name: MPI-ESM
           dataset_id: cmip5.output1.MPI-M.MPI-ESM-LR.decadal1994.mon.ocean.Omon.r1i1p1.v20120529|esgf1.dkrz.de
           variable_name: zos

http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/RCMES/run_RCMES.py
----------------------------------------------------------------------
diff --git a/RCMES/run_RCMES.py b/RCMES/run_RCMES.py
index f07f2ef..65d5e3c 100644
--- a/RCMES/run_RCMES.py
+++ b/RCMES/run_RCMES.py
@@ -30,28 +30,29 @@ from ocw.dataset import Bounds
 from ocw.dataset_loader import DatasetLoader
 from metrics_and_plots import *
 
-def load_datasets_from_config(*loader_options, **kwargs):
+def load_datasets_from_config(extra_opts, *loader_opts):
     '''
     Generic dataset loading function.
     '''
-    for opt in loader_options:
+    for opt in loader_opts:
         loader_name = opt['loader_name']
-        if loader_name == 'ESGF':
-            if password is None:
-                username=raw_input('Enter your ESGF OpenID:\n')
-                password=getpass(prompt='Enter your ESGF password:\n')
-
-            opt['username'] = username
-            opt['password'] = password
+        if loader_name == 'esgf':
+            if extra_opts['password'] is None:
+                extra_opts['username'] = raw_input('Enter your ESGF OpenID:\n')
+                extra_opts['password'] = getpass(
+                                        prompt='Enter your ESGF password:\n')
+
+            opt['esgf_username'] = extra_opts['username']
+            opt['esgf_password'] = extra_opts['password']
         elif loader_name == 'rcmed':
-            opt['min_lat'] = kwargs['min_lat']
-            opt['max_lat'] = kwargs['max_lat']
-            opt['min_lon'] = kwargs['min_lon']
-            opt['max_lon'] = kwargs['max_lon']
-            opt['start_time'] = kwargs['start_time']
-            opt['end_time'] = kwargs['end_time']
-
-        loader = DatasetLoader(*loader_options)
+            opt['min_lat'] = extra_opts['min_lat']
+            opt['max_lat'] = extra_opts['max_lat']
+            opt['min_lon'] = extra_opts['min_lon']
+            opt['max_lon'] = extra_opts['max_lon']
+            opt['start_time'] = extra_opts['start_time']
+            opt['end_time'] = extra_opts['end_time']
+
+        loader = DatasetLoader(*loader_opts)
         loader.load_datasets()
         return loader.datasets
 
@@ -75,8 +76,11 @@ min_lat = space_info['min_lat']
 max_lat = space_info['max_lat']
 min_lon = space_info['min_lon']
 max_lon = space_info['max_lon']
-kwargs = {'min_lat': min_lat, 'max_lat': max_lat, 'min_lon': min_lon,
-          'max_lon': max_lon, 'start_time': start_time, 'end_time': end_time}
+
+# Additional arguments for the DatasetLoader
+extra_opts = {'min_lat': min_lat, 'max_lat': max_lat, 'min_lon': min_lon,
+              'max_lon': max_lon, 'start_time': start_time,
+              'end_time': end_time, 'username': None, 'password': None}
 
 # Get the dataset loader options
 obs_data_info = config['datasets']['reference']
@@ -98,7 +102,7 @@ for i, info in enumerate(model_data_info):
 
 """ Step 1: Load the observation data """
 print 'Loading observation datasets:\n',obs_data_info
-obs_datasets = load_datasets_from_config(*obs_data_info, **kwargs)
+obs_datasets = load_datasets_from_config(extra_opts, *obs_data_info)
 obs_names = [dataset.name for dataset in obs_datasets]
 for i, dataset in enumerate(obs_datasets):
     if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
@@ -109,7 +113,7 @@ for i, dataset in enumerate(obs_datasets):
         obs_dataset.values *= multiplying_factor[i]
 
 """ Step 2: Load model NetCDF Files into OCW Dataset Objects """
-model_datasets = load_datasets_from_config(*model_data_info, **kwargs)
+model_datasets = load_datasets_from_config(extra_opts, *model_data_info)
 model_names = [dataset.name for dataset in model_datasets]
 if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
     for i, dataset in enumerate(model_datasets):

http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/ocw/data_source/esgf.py
----------------------------------------------------------------------
diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py
index 3ea559f..ac802c0 100644
--- a/ocw/data_source/esgf.py
+++ b/ocw/data_source/esgf.py
@@ -30,7 +30,7 @@ from bs4 import BeautifulSoup
 import requests
 
 def load_dataset(dataset_id,
-                 variable,
+                 variable_name,
                  esgf_username,
                  esgf_password,
                  search_url=DEFAULT_ESGF_SEARCH,
@@ -43,8 +43,8 @@ def load_dataset(dataset_id,
     :param dataset_id: The ESGF ID of the dataset to load.
     :type dataset_id: :mod:`string`
 
-    :param variable: The variable to load.
-    :type variable: :mod:`string`
+    :param variable_name: The variable to load.
+    :type variable_name: :mod:`string`
 
     :param esgf_username: ESGF OpenID value to use for authentication.
     :type esgf_username: :mod:`string`
@@ -79,7 +79,7 @@ def load_dataset(dataset_id,
     '''
     download_data = _get_file_download_data(url=search_url,
                                             dataset_id=dataset_id,
-                                            variable=variable)
+                                            variable=variable_name)
 
     datasets = []
     for url, var in download_data:
@@ -97,7 +97,7 @@ def load_dataset(dataset_id,
     origin = {
         'source': 'esgf',
         'dataset_id': dataset_id,
-        'variable': variable
+        'variable': variable_name
     }
     for ds in datasets:
         ds.origin = origin

http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/ocw/esgf/logon.py
----------------------------------------------------------------------
diff --git a/ocw/esgf/logon.py b/ocw/esgf/logon.py
index 031e968..8b4c034 100644
--- a/ocw/esgf/logon.py
+++ b/ocw/esgf/logon.py
@@ -32,16 +32,12 @@ def logon(openid, password):
     The certificate is written in the location ~/.esg/credentials.pem.
     The trusted CA certificates are written in the directory ~/.esg/certificates.
     '''
-    
     # Must configure the DN of the JPL MyProxy server if using a JPL openid
     if JPL_HOSTNAME in openid:
         os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN
 
     lm = LogonManager()
 
-    lm.logon_with_openid(openid, password)
+    lm.logon_with_openid(openid, password, bootstrap=True)
 
     return lm.is_logged_on()
-    
-    
-    


Mime
View raw message