climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dnad...@apache.org
Subject git commit: Add Resources sheet into excel spreadsheet. It is no longer necessary to have ".rc" file Add -x option in obs4Mips.py in the "file_template" it is now possible to specify the variable to pass a a parameter. "trmm_%s.lst, years" where years c
Date Tue, 30 Sep 2014 19:51:04 GMT
Repository: climate
Updated Branches:
  refs/heads/master 8469bcacb -> 7a0a7e2d4


Add Resources sheet into excel spreadsheet.  It is no longer necessary to have ".rc" file
Add -x option in obs4Mips.py
in the "file_template" it is now possible to specify the variable to pass a a parameter. 
"trmm_%s.lst,years" where years contain the values to pass into the format "%s".


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/7a0a7e2d
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/7a0a7e2d
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/7a0a7e2d

Branch: refs/heads/master
Commit: 7a0a7e2d4b52ab7573787425351999f7ab6d90ce
Parents: 8469bca
Author: Nadeau <dnadeau@GSSLA40018857.nccs.nasa.gov>
Authored: Tue Sep 30 15:43:30 2014 -0400
Committer: Nadeau <dnadeau@GSSLA40018857.nccs.nasa.gov>
Committed: Tue Sep 30 15:43:30 2014 -0400

----------------------------------------------------------------------
 obs4MIPs/INSTALLATION                           |   3 +
 obs4MIPs/README                                 |   2 +
 obs4MIPs/Toolbox/ESGFexcel.py                   | 125 +++++++++++++++++++
 .../examples/ECMWF/ecmwf_table_obs4MIPs.xls     | Bin 10752 -> 55296 bytes
 obs4MIPs/obs4MIPs_process.py                    |  29 +++--
 5 files changed, 152 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/INSTALLATION
----------------------------------------------------------------------
diff --git a/obs4MIPs/INSTALLATION b/obs4MIPs/INSTALLATION
index 082b787..e647d55 100644
--- a/obs4MIPs/INSTALLATION
+++ b/obs4MIPs/INSTALLATION
@@ -54,6 +54,9 @@ http://code.google.com/p/netcdf4-python/
 
 RESSOURCE file documentation
 =============================
+**********************
+*** NOTE:  The Resource file can now be including in the Excel spreadsheet as a new sheet
called: Resources.  See "ecmwf_table_obs4MIPs.xls" for details.
+**********************
 
 In order to run osb4MIPs, the first thing to do is to create a resource file giving information
about your original data. The resource files contain all Global Attributes and information
needed to convert a variable to a CMIP5 variable.  
 

http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/README
----------------------------------------------------------------------
diff --git a/obs4MIPs/README b/obs4MIPs/README
index f9983b8..c0cc05f 100644
--- a/obs4MIPs/README
+++ b/obs4MIPs/README
@@ -12,6 +12,8 @@ obs4MIPs.py -r trmm.rc
 
 Convert trmm data to CMIP5 format using excel spreadsheet 
 obs4MIPs.py -r ecmwf.rc
+or 
+obs4MIPs.py -x ecmwf_table_obs4MIPs.xls
 
 --------------
 Installation 

http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/Toolbox/ESGFexcel.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/Toolbox/ESGFexcel.py b/obs4MIPs/Toolbox/ESGFexcel.py
new file mode 100644
index 0000000..92fd4df
--- /dev/null
+++ b/obs4MIPs/Toolbox/ESGFexcel.py
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import pdb
+import xlrd
+import shutil
+import os,sys
+from numpy import arange
+from Toolbox.CMORresources import CMORAttributes
+
+#  ********************************************************************
+#         ESGFexcel()
+#
+#  ********************************************************************
+class ESGFexcel:
+    '''
+    Create dictionary based on a file using key=value standard from Excel
+    Sheet resource Key is in Column A, Value is in Column B.     
+    '''
+    def __init__(self,excelFile):
+        '''
+        '''
+        self.xcl = excelFile
+        if( os.path.isfile(self.xcl) ):
+                wb=xlrd.open_workbook(self.xcl)
+        else:
+           print "****** Could not find "+self.xcl+" file ****"
+           print "****** Please check excel file name ****"
+           raise NameError(self.xcl)
+
+        sheet=wb.sheet_by_name('Resources')
+
+        for i in  arange(sheet.nrows-1) + 1: 
+            key = sheet.row(i)[0]
+            value  = sheet.row(i)[1]
+
+        self.resources = dict( [( key, value, ) 
+                               for( key, value ) in 
+                               [ (sheet.row(i)[0].value, str(sheet.row(i)[1].value)) 
+                                 for i in  arange(sheet.nrows-1) + 1] ] )
+        pdb.set_trace()
+        self.ReadXCL()
+        print self.resources.keys()
+
+    def ReadXCL(self):
+        '''
+        Read Excel Table and fill rc variable related field.
+        '''
+        try:
+           import xlrd
+        except:
+           print "****** Could not find xlrd Python Package ****"
+           print "****** Please install xlrd package to read excel files ****"
+
+        if( os.path.isfile(self.xcl) ):
+                wb=xlrd.open_workbook(self.xcl)
+        else:
+           print "****** Could not find "+self.xcl+" file ****"
+           print "****** Please check excel file name ****"
+           raise NameError(self.xcl)
+
+        sheet=wb.sheet_by_name('Variables')
+
+        self.resources['cmor_var']    = [ sheet.row( i )[ 0 ].value.\
+                                          encode('ascii','ignore') \
+                                          for i in arange(sheet.nrows-2) + 2 ]
+
+        self.resources['original_var'] = [ sheet.row( i )[ 1 ].value.\
+                                           encode('ascii','ignore') \
+                                           for i in arange(sheet.nrows-2) + 2 ]
+
+        self.resources['original_units']=[ sheet.row( i )[ 2 ].value.\
+                                           encode('ascii','ignore') \
+                                           for i in arange(sheet.nrows-2) + 2 ]
+
+        self.resources['level']         =[ sheet.row( i )[ 3 ].value.\
+                                           encode('ascii','ignore') \
+                                           for i in arange(sheet.nrows-2) + 2]
+
+        self.resources['equation']     =[ sheet.row( i )[ 6 ].value.\
+                                           encode('ascii','ignore') \
+                                           for i in arange(sheet.nrows-2) + 2 ]
+
+        # -----------------------------------------------------------------
+        # Make sure it is a string. The main program will call eval on it.
+        # -----------------------------------------------------------------
+        self.resources['cmor_var']       = str(self.resources['cmor_var'] )
+        self.resources['original_var']   = str(self.resources['original_var'])
+        self.resources['original_units'] =str(self.resources['original_units'])
+        self.resources['equation']       =str(self.resources['equation'])
+        self.resources['level']          =str(self.resources['level'])
+        return 1
+
+    def __getitem__(self,key):
+        '''
+        Retreive item from resource dictionary
+        '''
+        return self.resources[key]
+
+    def __setitem__(self,key,value):
+        '''
+        '''
+        self.resources[key]=value
+
+    def __delete__(self,key):
+        '''
+        '''
+        del self.resources[key]
+
+

http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
----------------------------------------------------------------------
diff --git a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
index 271c2a7..d2d12d7 100644
Binary files a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls and b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls
differ

http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/obs4MIPs_process.py
----------------------------------------------------------------------
diff --git a/obs4MIPs/obs4MIPs_process.py b/obs4MIPs/obs4MIPs_process.py
index bb6d238..3370100 100755
--- a/obs4MIPs/obs4MIPs_process.py
+++ b/obs4MIPs/obs4MIPs_process.py
@@ -26,6 +26,7 @@ import factory
 import numpy
 from factory.formats import import_equation
 from Toolbox.ESGFresources import *
+from Toolbox.ESGFexcel import *
 from Toolbox.CMORresources import CMORTable
 
 
@@ -38,11 +39,18 @@ def process( rc ):
     '''
     Convert netcdf/matlab/grads files into CMIP5 format.
     '''
-    #pdb.set_trace()
+    pdb.set_trace()
     # ----------------------------
     #  Loop yearly on file list.  
     # ----------------------------
-    for year in rc['years'].split(","):
+    file_template = rc['file_template'].split(",");
+    if( len(file_template) == 2 ):
+        template_parameter = file_template[1]
+        rc['file_template'] = file_template[0]
+    else:
+        template_parameter = 'years'
+
+    for year in rc[template_parameter].split(","):
         if(year == ''):
             files= os.popen( "ls " + rc['file_template'] ).readlines()
         else:
@@ -419,9 +427,10 @@ def usage(message):
 def main():
     '''
     '''
+    pdb.set_trace()
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "hy:r:",
-                                   ["help" ,"year=","resource="])
+        opts, args = getopt.getopt(sys.argv[1:], "hy:r:x:",
+                                   ["help" ,"year=","resource=","excel="])
     except getopt.GetoptError, err:
         usage(str(err))# will print something like "option -a not recognized"
         return(2)
@@ -431,9 +440,12 @@ def main():
     # --------------------------
     year     = -1
     resource = None
+    excel    = None
     for o, a in opts:
         if o in ("-r", "--resource"):
             resource = a
+        elif o in ("-x", "--excel"):
+            excel = a
         elif o in ("-h", "--help"):
             usage()
             return(0)
@@ -445,14 +457,17 @@ def main():
     # ------------------------------
     # Does the resource file exist?
     # ------------------------------
-    if( resource == None ) or ( not os.path.isfile( resource ) ) :
-        usage("bad Input Resource File")
+    if( ((resource == None ) or ( not os.path.isfile( resource ) )) and (( excel == None
) or ( not os.path.isfile( excel ) )) ):
+        usage("bad Input Resource/Excel File")
         return 1
 
     # -----------------------
     # Read in "rc" file
     # -----------------------
-    rc=ESGFresources(resource)
+    if( resource ):
+       rc = ESGFresources( resource )
+    if( excel ):
+       rc = ESGFexcel( excel )
 
     # --------------------------------
     # Extract CMIP5 Table information


Mime
View raw message