oodt-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Mattmann, Chris A (388J)" <chris.a.mattm...@jpl.nasa.gov>
Subject Re: svn commit: r1303182 - in /oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps: config/ extractors/ util/
Date Tue, 20 Mar 2012 23:14:42 GMT
Hey Luca,

One thing I caught here was that there was a commented out CMDS url -- I think we should
remove that? If it's commented, looks extraneous...

Keep up the good work!

Cheers,
Chris

On Mar 20, 2012, at 11:44 PM, <luca@apache.org> wrote:

> Author: luca
> Date: Tue Mar 20 22:44:09 2012
> New Revision: 1303182
>
> URL: http://svn.apache.org/viewvc?rev=1303182&view=rev
> Log:
> Implementing expanded syntax for environment variable replacement (OODT-405)
>
> Modified:
>    oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
>    oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
>    oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
>    oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
>    oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
>    oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
(original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapConfigMetKeys.java
Tue Mar 20 22:44:09 2012
> @@ -69,5 +69,7 @@ public interface OpendapConfigMetKeys {
>   public static final String DATASET_MET_NAME_ATTR = "name";
>
>   public static final String DATASET_MET_VALUE_ATTR = "value";
> +
> +  public static final String RES_LOCATION_ATTR = "resLocation";
>
> }
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
(original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/config/OpendapProfileMetKeys.java
Tue Mar 20 22:44:09 2012
> @@ -26,6 +26,8 @@ public interface OpendapProfileMetKeys {
>
>         public static final String VARIABLES = "Variables";
>
> +        public static final String COORDINATES = "Coordinates";
> +
>         public static final String VARIABLES_LONG_NAMES = "Variable Long Names";
>
>         public static final String CF_STANDARD_NAMES = "CF Standard Names";
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
(original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/DasMetadataExtractor.java
Tue Mar 20 22:44:09 2012
> @@ -96,19 +96,39 @@ public class DasMetadataExtractor implem
>                        ProfileUtils.addIfNotExisting(metadata, key, att.getValues());
>                }
>
> -        // NetCDF variables
> +        // NetCDF coordinates
>         } else {
> -               // store variable name
> -               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES,
attName);
> -               // store "standard_name", "long_name"
> -               while (e.hasMoreElements()) {
> -                       String key = (String) e.nextElement();
> -                       Attribute att = at.getAttribute(key);
> -                       if (key.equalsIgnoreCase(STANDARD_NAME)) {
> -                               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES,
att.getValueAt(0));
> -                       } else if (key.equalsIgnoreCase(LONG_NAME)) {
> -                               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES,
att.getValueAt(0));
> -                       }
> +
> +               if (   attName.equalsIgnoreCase("lat") || attName.equalsIgnoreCase("latitude")
> +                               || attName.equalsIgnoreCase("lon") || attName.equalsIgnoreCase("longitude")
> +                               || attName.equalsIgnoreCase("time")
> +                               || attName.equalsIgnoreCase("alt") || attName.equalsIgnoreCase("altitude")
> +                               || attName.equalsIgnoreCase("depth")
> +                               ) {
> +
> +               // store coordinate name
> +               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.COORDINATES,
attName);
> +
> +               } else if (attName.startsWith("time_")) {
> +
> +                       // ignore for now - it's not a coordinate neither a variable
you would want to search on
> +
> +          // NetCDF variables
> +               } else {
> +
> +               // store variable name
> +               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES,
attName);
> +               // store "standard_name", "long_name"
> +               while (e.hasMoreElements()) {
> +                       String key = (String) e.nextElement();
> +                       Attribute att = at.getAttribute(key);
> +                       if (key.equalsIgnoreCase(STANDARD_NAME)) {
> +                               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.CF_STANDARD_NAMES,
att.getValueAt(0));
> +                       } else if (key.equalsIgnoreCase(LONG_NAME)) {
> +                               ProfileUtils.addIfNotNull(metadata, OpendapProfileMetKeys.VARIABLES_LONG_NAMES,
att.getValueAt(0));
> +                       }
> +               }
> +
>                }
>         }
>
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
(original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/extractors/ThreddsMetadataExtractor.java
Tue Mar 20 22:44:09 2012
> @@ -273,6 +273,8 @@ public class ThreddsMetadataExtractor im
>
>        // add opendap access URL
>        if (type.equalsIgnoreCase(ServiceType.OPENDAP.toString())) {
> +               // store opendap URL
> +               ProfileUtils.addIfNotNull(met,"OpendapUrl",url+".dods");
>          // note: special processing of opendap endpoints since URL in thredds catalog
is unusable without a suffix
>                ProfileUtils.addIfNotNull(met,"Access", this.encodeAccessTuple(url+".html",
ProfileUtils.MIME_TYPE_OPENDAP_HTML, type));
>        }
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
(original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileChecker.java
Tue Mar 20 22:44:09 2012
> @@ -82,6 +82,11 @@ public class ProfileChecker {
>         selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(),
ProfileUtils.MIME_TYPE_HTML),
>         true, sb);
>
> +    // <resLocation>http://cmds-gis.jpl.nasa.gov/Export/NetViewer.asp?Height=1024&amp;Width=1280&amp;BrowseImage=Aqua/MODIS
MHchla|application/gis|NetViewer</resLocation>
> +               ok = ok && checkResourceAttribute("Location of type "+ProfileUtils.MIME_TYPE_GIS,
> +        selectResourceLocationByMimeType(profile.getResourceAttributes().getResLocations(),
ProfileUtils.MIME_TYPE_GIS),
> +        true, sb);
> +
>                // <elemName>mission_name</elemName>
>                // <elemName>sensor</elemName>
>                // <elemName>...</elemName>
>
> Modified: oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
> URL: http://svn.apache.org/viewvc/oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java?rev=1303182&r1=1303181&r2=1303182&view=diff
> ==============================================================================
> --- oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
(original)
> +++ oodt/trunk/opendapps/src/main/java/org/apache/oodt/opendapps/util/ProfileUtils.java
Tue Mar 20 22:44:09 2012
> @@ -74,8 +74,9 @@ public class ProfileUtils {
>                // The delimiter must be a character that is not commonly used in the
metadata values,
>                // and that it does not a special regular expression character.
>                // Cannot use '#' as it is used in URL anchors, such as THREDDS urls.
> +               // Cannot user '?', '&' as they are used in URL query strings.
>                // Cannot use '|' as it is used as multi-part separators in encoding of
metadata fields.
> -               PathUtils.DELIMITER = "&";
> +               PathUtils.DELIMITER = "~";
>        }
>
>   // character separating multiple parts of the same metadata field,
> @@ -97,6 +98,7 @@ public class ProfileUtils {
>   public final static String MIME_TYPE_OPENDAP_DDS = "application/opendap-dds";
>   public final static String MIME_TYPE_OPENDAP_HTML = "application/opendap-html";
>   public final static String MIME_TYPE_RSS = "application/rss+xml";
> +  public final static String MIME_TYPE_GIS = "application/gis";
>
>
>   private static final Logger LOG = Logger.getLogger(ProfileUtils.class
> @@ -108,10 +110,19 @@ public class ProfileUtils {
>     for (ConstantSpec spec : conf.getConstSpecs()) {
>       if (spec.getType().equals(RES_ATTR_SPEC_TYPE)) {
>         try {
> -               String value = PathUtils.replaceEnvVariables(spec.getValue(), datasetMet,
true);
> -               if (StringUtils.hasText(value)) {
> -                       setResourceAttributes(resAttr, spec.getName(), value);
> +
> +               // first process expanded '[@...]' instructions
> +               List<String> values = multipleEnvVariablesReplacement(spec.getValue(),
datasetMet);
> +
> +               // then process standard '[...]' instructions
> +               for (String value : values) {
> +               String _value = PathUtils.replaceEnvVariables(value, datasetMet, true);
> +                       if (StringUtils.hasText(_value)) {
> +                               setResourceAttributes(resAttr, spec.getName(), _value);
> +                       }
> +
>                }
> +
>         } catch (Exception e) {
>           e.printStackTrace();
>           LOG.log(Level.WARNING, "Error setting field: [" + spec.getName()
> @@ -123,6 +134,44 @@ public class ProfileUtils {
>
>     return resAttr;
>   }
> +
> +  /**
> +   * Utility method to process environment replacement instructions of the form '[@key]'
> +   * resulting in as many output values as there are values for the environment variable
'key'.
> +   * Note that currently only one such pattern '[@key']' can be processed.
> +   *
> +   * @param value
> +   * @param metadata
> +   * @return
> +   */
> +  private static List<String> multipleEnvVariablesReplacement(String value, Metadata
metadata) {
> +
> +       List<String> newValues = new ArrayList<String>();
> +
> +       // regexp matching found > replace values
> +       int start = value.indexOf("[@");
> +       if (start>=0) {
> +
> +                       int end = value.indexOf("]",start+2);
> +                       // remove '[@',']' to obtain environment variable key
> +                       String envKey = value.substring(start+2,end);
> +               List<String> envValues = metadata.getAllMetadata(envKey);
> +               if (envValues!=null) {
> +               for (String envValue : envValues) {
> +                       // create new metadata value for this environment replacement
> +                       String newValue = value.replaceAll("\\[@"+envKey+"\\]", envValue);
> +                       newValues.add(newValue);
> +               }
> +               }
> +
> +    // regexp matching not found > return original value
> +       } else {
> +               newValues.add(value);
> +       }
> +
> +       return newValues;
> +
> +  }
>
>   public static ProfileAttributes getProfileAttributes(OpendapConfig conf, Metadata datasetMet)
{
>     ProfileAttributes profAttr = new ProfileAttributes();
>
>


++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Chris Mattmann, Ph.D.
Senior Computer Scientist
NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA
Office: 171-266B, Mailstop: 171-246
Email: chris.a.mattmann@nasa.gov
WWW:   http://sunset.usc.edu/~mattmann/
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Adjunct Assistant Professor, Computer Science Department
University of Southern California, Los Angeles, CA 90089 USA
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++


Mime
View raw message