climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ah...@apache.org
Subject svn commit: r1515633 - /incubator/climate/branches/RefactorPlots/rcmet/src/main/python/rcmes/services/run_rcmes_processing.py
Date Mon, 19 Aug 2013 22:05:24 GMT
Author: ahart
Date: Mon Aug 19 22:05:24 2013
New Revision: 1515633

URL: http://svn.apache.org/r1515633
Log:
CLIMATE-259: updates to the rcmes_go bottle service to incorporate generation of additional
plots

Modified:
    incubator/climate/branches/RefactorPlots/rcmet/src/main/python/rcmes/services/run_rcmes_processing.py

Modified: incubator/climate/branches/RefactorPlots/rcmet/src/main/python/rcmes/services/run_rcmes_processing.py
URL: http://svn.apache.org/viewvc/incubator/climate/branches/RefactorPlots/rcmet/src/main/python/rcmes/services/run_rcmes_processing.py?rev=1515633&r1=1515632&r2=1515633&view=diff
==============================================================================
--- incubator/climate/branches/RefactorPlots/rcmet/src/main/python/rcmes/services/run_rcmes_processing.py
(original)
+++ incubator/climate/branches/RefactorPlots/rcmet/src/main/python/rcmes/services/run_rcmes_processing.py
Mon Aug 19 22:05:24 2013
@@ -17,13 +17,26 @@
 #!/usr/local/bin/python
 """Module used to launch the RESTful API"""
 import sys
+import ast
 sys.path.append('../../.')
 from bottle import route, request
+
+from classes import GridBox, JobProperties, Model
+from storage import db
+from storage.rcmed import getParams
+from utils.misc import msg, readSubRegionsFile
+from toolkit.do_data_prep import prep_data
+from toolkit import process
+from toolkit.metrics import calculate_metrics_and_make_plots
+
+import ConfigParser
 import json
 import cli.do_rcmes_processing_sub as awesome
 import time
 import datetime
 import os
+import numpy as np
+import numpy.ma as ma
 time_format_new = '%Y-%m-%d %H:%M:%S'
 
 #Static Default params
@@ -37,6 +50,14 @@ maskLatMax=0         # float (only used 
 maskLonMin=0         # float (only used if maskOption=1)
 maskLonMax=0         # float (only used if maskOption=1)
 
+# Hard-coded configuration/settings values that 
+# are not yet mapped to UI settings
+precipFlag=False   # 
+maskOption=True    # To match rcmet.py line 221
+spatialGrid='user' # Eventually, use options['regrid']
+gridLonStep=0.5
+gridLatStep=0.5
+
 ###########################################################
 ##OPEN FOR DISCUSSION
 titleOption = 'default'   #this means that ' model vs obs :' will be used
@@ -45,84 +66,269 @@ plotFileNameOption = 'default'  #another
 
 @route('/rcmes/run/')
 def rcmes_go():
+
     print "**********\nBEGIN RCMES2.0_RUN\n**********"
-    print 'cachedir', cachedir
-    print 'workdir', workdir
     evalWorkDir = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
     evalPath = os.path.join( workdir, evalWorkDir )
     os.makedirs(evalPath)
     print 'evalPath', evalPath
     
+    # Attempt to create the cache dir if it does not exist`
     try:
         if not os.path.exists(cachedir):
             os.makedirs(cachedir)
     except Error as e:
         print "I/O error({0}: {1}".format(e.errno, e.strerror)
         sys.exit(1)
+    
+    # Print request variables as received
+    msg('Request Variables', {k:request.query.get(k) for k in request.query.keys()})
 
-    obsDatasetId = int(request.query.get('obsDatasetId', '').strip())
-    print 'obsDatasetId', obsDatasetId
-    obsParameterId = int(request.query.get('obsParameterId', '').strip())
-    print 'obsParameterId', obsParameterId
-
-    #reformat DateTime after pulling it out of the POST
-    POSTstartTime = str(request.query.get('startTime', '').strip())
-    startTime = datetime.datetime.fromtimestamp(time.mktime(time.strptime(POSTstartTime,
time_format_new)))
-    print 'startTime', startTime
-    #reformat DateTime after pulling it out of the POST
-    POSTendTime = str(request.query.get('endTime', '').strip())
-    endTime = datetime.datetime.fromtimestamp(time.mktime(time.strptime(POSTendTime, time_format_new)))
-    print 'endTime', endTime
+    # Obtain the observational dataset and parameter ids from the request
+    obsDatasetIds = ast.literal_eval(request.query.get('obsDatasetIds', '[]'))
+    obsParameterIds = ast.literal_eval(request.query.get('obsParameterIds', '[]'))
 
+
+    # Reformat start date/time after pulling it out of the request
+    requestStartTime = str(request.query.get('startTime', '').strip())
+    startTime = datetime.datetime.fromtimestamp(time.mktime(time.strptime(requestStartTime,
time_format_new)))
+    
+    # Reformat end date/time after pulling it out of the request
+    requestEndTime = str(request.query.get('endTime', '').strip())
+    endTime = datetime.datetime.fromtimestamp(time.mktime(time.strptime(requestEndTime, time_format_new)))
+
+    # Obtain the geographical boundaries from the request
     latMin = float(request.query.get('latMin', '').strip())
-    print 'latMin', latMin
     latMax = float(request.query.get('latMax', '').strip())
-    print 'latMax', latMax 
     lonMin = float(request.query.get('lonMin', '').strip())
-    print 'lonMin', lonMin
     lonMax = float(request.query.get('lonMax', '').strip())
-    print 'lonMax', lonMax
-
-    filelist = [request.query.get('filelist', '').strip()]
-    print 'filelist', filelist[0]
-
-    modelVarName = str(request.query.get('modelVarName', '').strip())
-    print 'modelVarName', modelVarName
-    precipFlag = request.query.get('precipFlag', '').strip()
-    print 'precipFlag', precipFlag
-    modelTimeVarName = str(request.query.get('modelTimeVarName', '').strip())
-    print 'modelTimeVarName', modelTimeVarName
-    modelLatVarName = str(request.query.get('modelLatVarName', '').strip())
-    print 'modelLatVarName', modelLatVarName
-    modelLonVarName = str(request.query.get('modelLonVarName', '').strip())
-    print 'modelLonVarName', modelLonVarName
 
+    # Obtain the list of model files from the request
+    filelist = ast.literal_eval(request.query.get('filelist', '[]'))
+    
+    # Obtain the evaluation variable name info for each model from the request
+    modelVarNames = ast.literal_eval(request.query.get('modelVarName', '[]'))
+    
+    # Obtain the time variable name info for each model from the request
+    modelTimeVarNames = ast.literal_eval(request.query.get('modelTimeVarName', '[]'))
+    
+    # Obtain the lat variable name info for each model from the request
+    modelLatVarNames = ast.literal_eval(request.query.get('modelLatVarName', '[]'))    
+    modelLonVarNames = ast.literal_eval(request.query.get('modelLonVarName', ''))
+    
+    # Obtain regrid configuration info from the request
     regridOption = str(request.query.get('regridOption', '').strip())
-    print 'regridOption', regridOption
     timeRegridOption = str(request.query.get('timeRegridOption', '').strip())
-    print 'timeRegridOption', timeRegridOption
+   
+    # Parse subregion information from the request
+    subRegionFile = str(request.query.get('subregionFile','').strip())
+   
+    # Parse additional options from the request
     seasonalCycleOption = request.query.get('seasonalCycleOption', '').strip()
-    print 'seasonalCycleOption', seasonalCycleOption
     metricOption = str(request.query.get('metricOption', '').strip())
-    print 'metricOption', metricOption    
     
-    settings = {"cacheDir": cachedir, "workDir": evalPath, "fileList": filelist}
-    params = {"obsDatasetId": obsDatasetId, "obsParamId": obsParameterId, 
-              "startTime": startTime, "endTime": endTime, "latMin": latMin, 
-              "latMax": latMax, "lonMin": lonMin, "lonMax": lonMax}
-    model = {"varName": modelVarName, "timeVariable": modelTimeVarName, 
-             "latVariable": modelLatVarName, "lonVariable": modelLonVarName}
-    mask = {"latMin": latMin, "latMax": latMax, "lonMin": lonMin, "lonMax": lonMax}
-    options = {"regrid": regridOption, "timeRegrid": timeRegridOption, 
-               "seasonalCycle": seasonalCycleOption, "metric": metricOption, 
-               "plotTitle": titleOption, "plotFilename": plotFileNameOption, 
-               "mask": maskOption, "precip": precipFlag}
-    
-    awesome.do_rcmes(settings, params, model, mask, options)
-    
-    model_path = os.path.join(evalPath, plotFileNameOption + "model.png")
-    obs_path = os.path.join(evalPath, plotFileNameOption + "obs.png")
-    comp_path = os.path.join(evalPath, plotFileNameOption + ".png")
+    
+    # Build dicts of extracted request information
+    settings = {"cacheDir": cachedir, "workDir": workdir, "fileList": filelist}
+    params   = {"obsDatasetIds": obsDatasetIds, "obsParamIds": obsParameterIds, 
+                "startTime": startTime, "endTime": endTime, "latMin": latMin, 
+                "latMax": latMax, "lonMin": lonMin, "lonMax": lonMax}
+    models   = {"varNames": modelVarNames, "timeVariables": modelTimeVarNames, 
+                "latVariables": modelLatVarNames, "lonVariables": modelLonVarNames}
+    mask     = {"latMin": latMin, "latMax": latMax, "lonMin": lonMin, 
+                "lonMax": lonMax}
+    options  = {"regrid": regridOption, "timeRegrid": timeRegridOption, 
+                "seasonalCycle": seasonalCycleOption, "metric": metricOption, 
+                "plotTitle": titleOption, "plotFilename": plotFileNameOption, 
+                "mask": maskOption, "precip": precipFlag}
+    
+    # Include optional information
+    options['subRegionFile'] = subRegionFile if subRegionFile != '' else False
+    
+    
+    # Summarize what was extracted from the request
+    msg('Parsed Evaluation Criteria: ---------')
+    msg('Settings',  settings, 2)
+    msg('Parameters',params, 2)
+    msg('Models',    models, 2)
+    msg('Mask',      mask, 2)
+    msg('Options',   options, 2)
+    
+    # Parse the provided subregion data
+    if options['subRegionFile']:
+        # Parse the Config file
+        subRegions = readSubRegionsFile(options['subRegionFile'])
+        msg("Parsed SubRegions", subRegions)
+    
+    # Create a JobProperties object with the information
+    # extracted from the request
+    jobProperties = JobProperties(
+        settings['workDir'],
+        settings['cacheDir'],
+        spatialGrid,
+        options['timeRegrid'],
+        latMin=params['latMin'], # only used if spatial grid ='user'
+        latMax=params['latMax'], # only used if spatial grid ='user'
+        lonMin=params['lonMin'], # only used if spatial grid ='user'
+        lonMax=params['lonMax'], # only used if spatial grid ='user'
+        startDate=params['startTime'].strftime("%Y%m%d"),
+        endDate=params['endTime'].strftime("%Y%m%d"))
+        
+        
+    # Create a GridBox object with the spatial information
+    # extracted from the request
+    gridBox = GridBox(params['latMin'],
+        params['lonMin'],
+        params['latMax'],
+        params['lonMax'],
+        gridLonStep,
+        gridLatStep)
+        
+    # Prepare requested model files
+    eval_models = []
+    for i in xrange(len(settings['fileList'])):
+        # use getModelTimes(modelFile,timeVarName) to generate the 
+        # modelTimeStep and time list
+        _ , timestep = process.getModelTimes(
+            settings['fileList'][i],
+            models['timeVariables'][i])
+            
+        modelInfo = {
+            'filename':     settings['fileList'][i],
+            'latVariable':  models['latVariables'][i],
+            'lonVariable':  models['lonVariables'][i],
+            'timeVariable': models['timeVariables'][i],
+            'varName':      models['varNames'][i],
+            'timeStep':     timestep,
+            'precipFlag':   options['precip']
+        }
+        
+        msg("Built model info dict for {0}".format(settings['fileList'][i]),
+            modelInfo)
+
+        model = Model(**modelInfo)
+        eval_models.append(model)
+        
+    msg("Prepared {0} models".format(len(eval_models)))
+    
+    # Prepare requested observational data
+    # Obtain dataset metadata from RCMED Query Service
+    obs_timesteps = []
+    for i in xrange(len(params['obsDatasetIds'])):
+        dId = params['obsDatasetIds'][i]
+        pId = params['obsParamIds'][i]
+        query_service_url= 'http://rcmes.jpl.nasa.gov/query-api/query.php?datasetId={0}&parameterId={1}'.format(dId,pId)
+        msg("Obtaining dataset information from", query_service_url)
+        obs_timesteps.append(db.get_param_info(query_service_url)[1])
+    
+    # Build dict of dataset metadata
+    obsInfo = {
+        'obsDatasetId': params['obsDatasetIds'],
+        'obsParamId':   params['obsParamIds'],
+        'obsTimeStep':  obs_timesteps
+    }
+    
+    # Get parameter listing from the database
+    params = getParams()
+    
+    # Build the list of observational datasets that exist in RCMED
+    eval_datasets = []
+    for param_id in obsInfo['obsParamId']:
+        for param in params:
+            if param['parameter_id'] == int(param_id):
+                eval_datasets.append(param)
+    
+    
+    msg("Observation Info", obsInfo)
+    msg("Observation Dataset List", eval_datasets)
+    
+
+    # Perform the requested evaluation
+    msg('Evaluation Output: ------------------')
+    numOBSs, numMDLs, nT, ngrdY, ngrdX, Times, lons, lats, obsData, modelData, obsList, mdlName
= prep_data(jobProperties,eval_datasets,gridBox,eval_models)
+    
+    msg('numOBSs',numOBSs)
+    msg('numMDLs',numMDLs)
+    msg('nT',nT)
+    msg('ngrdY',ngrdY)
+    msg('ngrdX',ngrdX)
+    msg('Times',Times)
+    msg('lons',lons)
+    msg('lats',lats)
+    msg('obsData Length',len(obsData))
+    msg('modelData Length',len(modelData))
+    msg('obsList',obsList)
+    msg('mdlName',mdlName)
+    
+    # Prepare SubRegion data structures
+    if options['subRegionFile']:
+        numSubRgn = len(subRegions)
+        msg("Processing {0} sub regions...".format(numSubRgn))
+        if numSubRgn > 0:
+            subRgnName = [ x.name   for x in subRegions ]
+            subRgnLon0 = [ x.lonMin for x in subRegions ]
+            subRgnLon1 = [ x.lonMax for x in subRegions ]
+            subRgnLat0 = [ x.latMin for x in subRegions ]
+            subRgnLat1 = [ x.latMax for x in subRegions ]
+            # compute the area-mean timeseries for all subregions.
+            #   the number of subregions is usually small and memory usage 
+            #   is usually not a concern
+            obsRgn = ma.zeros((numOBSs, numSubRgn, nT))
+            mdlRgn = ma.zeros((numMDLs, numSubRgn, nT))
+            
+            print 'Enter area-averaging: mdlData.shape, obsData.shape ', modelData.shape,
obsData.shape
+            print 'Use Latitude/Longitude Mask for Area Averaging'
+            for n in np.arange(numSubRgn):
+                # Define mask using regular lat/lon box specified by users ('mask=True' defines
the area to be excluded)
+                maskLonMin = subRgnLon0[n] 
+                maskLonMax = subRgnLon1[n]
+                maskLatMin = subRgnLat0[n]
+                maskLatMax = subRgnLat1[n]
+                mask = np.logical_or(np.logical_or(lats <= maskLatMin, lats >= maskLatMax),

+                                    np.logical_or(lons <= maskLonMin, lons >= maskLonMax))
+                # Calculate area-weighted averages within this region and store in a new
list
+                for k in np.arange(numOBSs):           # area-average obs data
+                    Store = []
+                    for t in np.arange(nT):
+                        Store.append(process.calc_area_mean(obsData[k, t, :, :], lats, lons,
mymask = mask))
+                    obsRgn[k, n, :] = ma.array(Store[:])
+                for k in np.arange(numMDLs):           # area-average mdl data
+                    Store = []
+                    for t in np.arange(nT):
+                        Store.append(process.calc_area_mean(modelData[k, t, :, :], lats,
lons, mymask = mask))
+                    mdlRgn[k, n, :] = ma.array(Store[:])
+                Store = []                               # release the memory allocated by
temporary vars
+    
+    # Call combined metrics/plotting function
+    print options['subRegionFile']
+    print options['subRegionFile'] == False
+    calculate_metrics_and_make_plots(
+        models['varNames'][0], # won't work if models have different var names among themselves
+        settings['workDir'],
+        lons, 
+        lats, 
+        obsData, 
+        modelData, 
+        (obsRgn if options['subRegionFile'] != False else None),  # Depends on presence of
subregion info
+        (mdlRgn if options['subRegionFile'] != False else None),  # Depends on presence of
subregion info
+        obsList, 
+        mdlName, 
+        (True if options['subRegionFile'] != False else False),   # Depends on presence of
subregion info
+        (subRgnLon0 if options['subRegionFile'] != False else None), 
+        (subRgnLon1 if options['subRegionFile'] != False else None), 
+        (subRgnLat0 if options['subRegionFile'] != False else None), 
+        (subRgnLat1 if options['subRegionFile'] != False else None))
+        
+    # At this point, all of the plots have been written to various files
+    
+    # Obsolete invocation
+    #awesome.do_rcmes(settings, params, models, mask, options)
+    
+    # Prepare a response with the results of the evaluation
+    msg('Evaluation Complete, Preparing Response...')
+    model_path = os.path.join(workdir, plotFileNameOption + "model.png")
+    obs_path = os.path.join(workdir,   plotFileNameOption + "obs.png")
+    comp_path = os.path.join(workdir,  plotFileNameOption + ".png")
 
     product_dict = {'modelPath':model_path,
                     'obsPath': obs_path,



Mime
View raw message