oodt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bfos...@apache.org
Subject svn commit: r1052149 [1/2] - in /oodt/branches/wengine-branch/pge: ./ src/ src/main/ src/main/assembly/ src/main/bin/ src/main/java/ src/main/java/gov/ src/main/java/gov/nasa/ src/main/java/gov/nasa/jpl/ src/main/java/gov/nasa/jpl/oodt/ src/main/java/g...
Date Thu, 23 Dec 2010 02:48:37 GMT
Author: bfoster
Date: Thu Dec 23 02:48:35 2010
New Revision: 1052149

URL: http://svn.apache.org/viewvc?rev=1052149&view=rev
Log:
wengine branch

Added:
    oodt/branches/wengine-branch/pge/
    oodt/branches/wengine-branch/pge/CHANGES.txt
    oodt/branches/wengine-branch/pge/LICENSE.txt
    oodt/branches/wengine-branch/pge/pom.xml
    oodt/branches/wengine-branch/pge/src/
    oodt/branches/wengine-branch/pge/src/main/
    oodt/branches/wengine-branch/pge/src/main/assembly/
    oodt/branches/wengine-branch/pge/src/main/assembly/assembly.xml
    oodt/branches/wengine-branch/pge/src/main/bin/
    oodt/branches/wengine-branch/pge/src/main/bin/pgetask   (with props)
    oodt/branches/wengine-branch/pge/src/main/java/
    oodt/branches/wengine-branch/pge/src/main/java/gov/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/jpl/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/jpl/oodt/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/jpl/oodt/cas/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/jpl/oodt/cas/pge/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/jpl/oodt/cas/pge/query/
    oodt/branches/wengine-branch/pge/src/main/java/gov/nasa/jpl/oodt/cas/pge/writers/
    oodt/branches/wengine-branch/pge/src/main/java/org/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/ConfigFilePropertyAdder.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/StdPGETaskInstance.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/PGETaskWorkflowCondition.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/WorkflowConditionPropAdder.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/DynamicConfigFile.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileBasedPgeConfigBuilder.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfig.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigBuilder.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/RegExprOutputFiles.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/RenamingConv.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/XmlFilePgeConfigBuilder.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/metadata/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PGETaskMetKeys.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/query/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/query/conv/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/query/conv/DateVersionConverter.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/staging/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileManagerFileStager.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/PcsMetFileWriter.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/SciPgeConfigFileWriter.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/metlist/MetadataListPcsMetFileWriter.java
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/
    oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java
    oodt/branches/wengine-branch/pge/src/main/resources/
    oodt/branches/wengine-branch/pge/src/main/resources/REMOVE.log
    oodt/branches/wengine-branch/pge/src/main/resources/examples/
    oodt/branches/wengine-branch/pge/src/main/resources/examples/MetadataOutputFiles/
    oodt/branches/wengine-branch/pge/src/main/resources/examples/MetadataOutputFiles/metadata-output.xml   (with props)
    oodt/branches/wengine-branch/pge/src/main/resources/examples/PgeConfigFiles/
    oodt/branches/wengine-branch/pge/src/main/resources/examples/PgeConfigFiles/pge-config.xml   (with props)
    oodt/branches/wengine-branch/pge/src/main/resources/examples/WorkflowMgrInput/
    oodt/branches/wengine-branch/pge/src/main/resources/examples/WorkflowMgrInput/pcs-pge-dyn-metadata.met
    oodt/branches/wengine-branch/pge/src/main/resources/examples/WorkflowMgrInput/pcs-pge-task-workflow.properties   (with props)
    oodt/branches/wengine-branch/pge/src/main/resources/examples/XsltFiles/
    oodt/branches/wengine-branch/pge/src/main/resources/examples/XsltFiles/xslt_prop_eq_val.xml
    oodt/branches/wengine-branch/pge/src/main/resources/logging.properties
    oodt/branches/wengine-branch/pge/src/site/
    oodt/branches/wengine-branch/pge/src/site/resources/
    oodt/branches/wengine-branch/pge/src/site/resources/images/
    oodt/branches/wengine-branch/pge/src/site/resources/images/cas-pge.jpg   (with props)
    oodt/branches/wengine-branch/pge/src/site/resources/images/cas-pge.psd   (with props)
    oodt/branches/wengine-branch/pge/src/test/
    oodt/branches/wengine-branch/pge/src/test/gov/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/jpl/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/jpl/oodt/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/jpl/oodt/cas/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/jpl/oodt/cas/pge/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/jpl/oodt/cas/pge/util/
    oodt/branches/wengine-branch/pge/src/test/gov/nasa/jpl/oodt/cas/pge/util/UtilsTest.java

Added: oodt/branches/wengine-branch/pge/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/CHANGES.txt?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/CHANGES.txt (added)
+++ oodt/branches/wengine-branch/pge/CHANGES.txt Thu Dec 23 02:48:35 2010
@@ -0,0 +1,83 @@
+CAS PGE Framework Changelog
+
+
+Major Releases:
+===========================================
+1.2.0 Current Development (Unreleased Changes)
+
+- OODT-373 Mutilple PcsMetFileWriters should be able run on each output product file (bfoster)
+
+- OODT-298 Allow cas-crawler to have global metadata (bfoster)
+
+- OODT-299 allow cas-pge XmlFilePgeConfigBuilder and MetadataListPcsMetFileWriter xml file metadata tags accept values through val attr and as text_content (bfoster)
+
+- OODT-296 Add ability for cas-pge to do automated product file renaming before met file creation and ingest (bfoster)
+
+- OODT-295 Allow PgeMetadata to be able to be merged with another PgeMetadata AND remove envReplace on convertion of static-workflow metadata inside of PgeMetadata (bfoster)
+
+- OODT-294 Make cas-pge's extra separate log file handle only log pge stream output instead of all cas-pge logging (bfoster)
+
+- OODT-293 Fix Namespacing in cas-pges XmlFilePgeConfigBuilder (bfoster)
+
+- OODT-281 Upgrades to cas-metadata-1.6.1, cas-commons-1.1.1, cas-filemgr-1.7.2, 
+           cas-workflow-1.5.1 and cas-crawler-2.2.1 (mattmann)
+
+- OODT-270 Update pom to include saxon-dom jar dependency (bfoster)
+
+
+1.1.2 - 12/03/2008
+
+- OODT-281 Upgrades to cas-metadata-1.6.1, cas-commons-1.1.1, cas-filemgr-1.7.2, 
+           cas-workflow-1.5.1 and cas-crawler-2.2.1 (mattmann)
+
+- OODT-270 Update pom to include saxon-dom jar dependency (bfoster)
+
+1.1.0 - 8/23/2008
+
+- added support for * for ProductTypes in FROM value for SqlQuery (bfoster)
+
+- XmlFilePgeConfigBuilder import tags is now able to handle recursive import of parents of more than one hierarchy up (bfoster)
+
+- MetadataListPcsMetFileWriter now accepts envReplace and split args in its metadata tags (bfoster)
+
+- OODT-249 Code that uses the cas-filemgr should only have to be aware of an Element's name (bfoster)
+
+- OODT-246  cas-pge log handler fails to readd for each run after first run (bfoster)
+
+- Created query tool which uses SqlQuery type queries (bfoster)
+
+- Refactoring of SqlQuery utilities (bfoster)
+
+- OODT-237 Make cas-pge log file deletable when running in workflow (bfoster)
+
+- OODT-232 Modify XmlFilePgeConfigBuilder so that is can add metadata to the workflow dynamic metadata (bfoster)
+
+- OODT-235 Move cas-query classes into cas-pge (bfoster)
+
+- OODT-230 Allow xml PgeConfig xml files to import setting from other PgeConfig xml files (bfoster)
+
+1.0.0 - 08/01/2008
+
+- OODT-199 Separate PEATE specific stuff out of OODT projects into the peate directory branch (bfoster)
+
+- OODT-221 Allow CAS-PGE to be able to log its output to a separate log file, while still logging to current mediums (bfoster)
+
+- OODT-184 Upgrade to JDK5 (mattmann)
+
+- OODT-185 Upgrade to Maven2 (mattmann)
+
+- OODT-203 Allow CAS-PGE to be able to update dynamic metadata from workflow manager (bfoster)
+
+- OODT-202 Modify PgeConfig to be built from a builder interface, not just from an xml file (bfoster)
+
+- formatting updates (mattmann)
+
+- OODT-183 package naming scheme for cas-pge should be changed to *.pge (bfoster)
+
+- OODT-195 Upgrade to cas-crawler-2.0-dev (mattmann)
+
+- OODT-194 Update to work with new cas-metadata 1.4.0-dev (bfoster)
+
+- initial import out of OCO CM (mattmann,bfoster)
+
+- cleanup of project structure and formatting (mattmann)

Added: oodt/branches/wengine-branch/pge/LICENSE.txt
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/LICENSE.txt?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/LICENSE.txt (added)
+++ oodt/branches/wengine-branch/pge/LICENSE.txt Thu Dec 23 02:48:35 2010
@@ -0,0 +1,7 @@
+Copyright 1999-2008 California Institute of Technology. ALL RIGHTS RESERVED. U.S. Government Sponsorship acknowledged. REDISTRIBUTION AND/OR MODIFICATION AND/OR USE OF THIS SOFTWARE WITHOUT PRIOR AUTHORIZATION IS STRICTLY FORBIDDEN.
+
+Recipient, its agents, assigns, and associates may not grant rights to another or others with respect to the Software. Any such attempt or use for public or commercial purposes will automatically terminate the rights granted under this License and will be sufficient impetus for the California Institute of Technology and the United States Government to bring joint and swift legal action and/or terminal effects.
+
+THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, THAT THE SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE SOFTWARE IS WITH RECIPIENT. SHOULD ANY SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, RECIPIENT ASSUMES THE COST OF ANY NECESSARY SERVICING, REPAIR, OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
+
+This software product includes software developed by the Apache Software Foundation (http://www.apache.org/).

Added: oodt/branches/wengine-branch/pge/pom.xml
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/pom.xml?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/pom.xml (added)
+++ oodt/branches/wengine-branch/pge/pom.xml Thu Dec 23 02:48:35 2010
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.oodt</groupId>
+    <artifactId>oodt-core</artifactId>
+    <version>0.2-incubating</version>
+  </parent>
+  <artifactId>cas-pge</artifactId>
+  <name>CAS PGE Adaptor Framework</name>
+  <version>0.2-branch</version>
+  <description>Allows data processing jobs not written in comformance with the
+        PCS PGE interface to be run within the PCS.</description>
+  <ciManagement>
+    <notifiers>
+      <notifier>
+        <configuration >
+          <address>chris.mattmann@jpl.nasa.gov</address>
+        </configuration>
+      </notifier>
+    </notifiers>
+  </ciManagement>
+ <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.2-beta-2</version>
+        <configuration>
+          <descriptors>
+            <descriptor>src/main/assembly/assembly.xml</descriptor>
+          </descriptors>
+          <archive>
+            <manifest>
+              <mainClass>gov.nasa.jpl.oodt.cas.pge.PGETask</mainClass>
+            </manifest>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>    
+    </plugins>
+  </build>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>cas-metadata</artifactId>
+      <version>0.2-incubating</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>oodt-commons</artifactId>
+       <version>0.2-incubating</version>
+    </dependency>    
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>cas-filemgr</artifactId>
+      <version>0.2-branch</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>cas-wengine</artifactId>
+      <version>0.2-branch</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>cas-crawler</artifactId>
+      <version>0.2-incubating</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.oodt</groupId>
+          <artifactId>cas-workflow</artifactId>
+        </exclusion> 
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
+      <version>2.3</version>
+    </dependency>
+    <dependency>
+      <groupId>net.sf.saxon</groupId>
+      <artifactId>saxon-dom</artifactId>
+      <version>8.7</version>      
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.2</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

Added: oodt/branches/wengine-branch/pge/src/main/assembly/assembly.xml
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/assembly/assembly.xml?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/assembly/assembly.xml (added)
+++ oodt/branches/wengine-branch/pge/src/main/assembly/assembly.xml Thu Dec 23 02:48:35 2010
@@ -0,0 +1,66 @@
+<!-- 
+Copyright 2008 California Institute of Technology. ALL RIGHTS
+RESERVED. U.S. Government Sponsorship acknowledged.
+
+$Id$
+-->
+<assembly>
+  <id>dist</id>
+  <formats>
+    <format>tar.gz</format>
+    <format>zip</format>
+  </formats>
+  <includeBaseDirectory>true</includeBaseDirectory>
+  <baseDirectory>${project.artifactId}-${project.version}</baseDirectory>
+  <includeSiteDirectory>false</includeSiteDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>${basedir}</directory>
+      <outputDirectory>.</outputDirectory>
+      <includes>
+        <include>LICENSE.txt</include>
+        <include>CHANGES.txt</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/bin</directory>
+      <outputDirectory>bin</outputDirectory>
+      <includes/>
+      <fileMode>755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/resources</directory>
+      <outputDirectory>etc</outputDirectory>
+      <includes>
+        <include>logging.properties</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/resources</directory>
+      <outputDirectory>logs</outputDirectory>
+      <includes>
+        <include>REMOVE.log</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/resources/examples</directory>
+      <outputDirectory>etc/examples</outputDirectory>
+      <excludes/>
+    </fileSet>
+    <fileSet>
+      <directory>target/site/apidocs</directory>
+      <filtered>false</filtered>
+      <outputDirectory>doc</outputDirectory>
+      <excludes/>
+    </fileSet>
+  </fileSets>
+  <dependencySets>
+    <dependencySet>
+      <outputDirectory>lib</outputDirectory>
+      <unpack>false</unpack>
+      <useProjectArtifact>true</useProjectArtifact>
+      <useTransitiveDependencies>true</useTransitiveDependencies>
+      <unpackOptions/>
+    </dependencySet>
+  </dependencySets>
+</assembly>

Added: oodt/branches/wengine-branch/pge/src/main/bin/pgetask
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/bin/pgetask?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/bin/pgetask (added)
+++ oodt/branches/wengine-branch/pge/src/main/bin/pgetask Thu Dec 23 02:48:35 2010
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+$JAVA_HOME/bin/java -Djava.ext.dirs=../lib \
+        -Djava.util.logging.config.file=../etc/logging.properties \
+        -Djavax.xml.transform.TransformerFactory=net.sf.saxon.TransformerFactoryImpl \
+        org.apache.oodt.cas.pge.PGETask --metadata ../etc/examples/WorkflowMgrInput/pcs-pge-dyn-metadata.met \
+        --config ../etc/examples/WorkflowMgrInput/pcs-pge-task-workflow.properties \
+        --instanceClass org.apache.oodt.cas.pge.StdPGETaskInstance
\ No newline at end of file

Propchange: oodt/branches/wengine-branch/pge/src/main/bin/pgetask
------------------------------------------------------------------------------
    svn:executable = 

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/ConfigFilePropertyAdder.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/ConfigFilePropertyAdder.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/ConfigFilePropertyAdder.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/ConfigFilePropertyAdder.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge;
+
+//OODT imports
+import org.apache.oodt.cas.workflow.metadata.ControlMetadata;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * An interface for adding dynamic properties to the {@link Metadata} based on
+ * metadata supplied. This interface is used by the PGETask to dynamically add
+ * PGE-specific properties that aren't general to all PGEs.
+ * </p>
+ * 
+ */
+public interface ConfigFilePropertyAdder {
+
+    public void addConfigProperties(ControlMetadata crtlMetadata, Object... objs);
+
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge;
+
+//JDK imports
+import java.io.File;
+import java.util.Properties;
+import java.util.Map.Entry;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+
+/**
+ * 
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * Runs a {@link PGETaskInstance} given {@link Metadata} and a
+ * {@link WorkflowTaskConfiguration}
+ * </p>.
+ */
+public class PGETask {
+
+    private Metadata metadata;
+
+    private Properties wftConfig;
+
+    public PGETask(Metadata metadata, Properties wftConfig) {
+        this.metadata = metadata;
+        this.wftConfig = wftConfig;
+    }
+
+    public void run(String pgeTaskInstanceClasspath)
+            throws Exception {
+        PGETaskInstance pgeTaskInst = (PGETaskInstance) Class.forName(
+                pgeTaskInstanceClasspath).newInstance();
+        pgeTaskInst.setDynamicMetadata(this.metadata);
+        pgeTaskInst.setStaticMetadata(this.convertToMetadata(this.wftConfig));
+        pgeTaskInst.execute();
+    }
+
+    public static void main(String[] args) throws Exception {
+        String metadataFilePath = null, configPropertiesPath = null;
+        String pgeTaskInstanceClasspath = null;
+        String usage = "PGETask --instanceClass <PGETaskInstance> "
+                + "--metadata </path/to/metadata/file> "
+                + "--config </path/to/task/config/file>\n";
+
+        for (int i = 0; i < args.length; i++) {
+            if (args[i].equals("--metadata")) {
+                metadataFilePath = args[++i];
+            } else if (args[i].equals("--config")) {
+                configPropertiesPath = args[++i];
+            } else if (args[i].equals("--instanceClass")) {
+                pgeTaskInstanceClasspath = args[++i];
+            }
+        }
+
+        if (metadataFilePath == null || configPropertiesPath == null) {
+            System.err.println(usage);
+            System.exit(1);
+        }
+
+        SerializableMetadata sm = new SerializableMetadata("UTF-8", false);
+        sm.loadMetadataFromXmlStream(new File(metadataFilePath).toURL()
+                .openStream());
+        Properties config = new Properties();
+        config.load(new File(configPropertiesPath).toURL().openStream());
+
+        PGETask task = new PGETask(sm, config);
+        task.run(pgeTaskInstanceClasspath);
+    }
+    
+	private Metadata convertToMetadata(Properties config) throws Exception {
+		Metadata metadata = new Metadata();
+		for (Entry entry : config.entrySet())
+			metadata.replaceMetadata((String) entry.getKey(), (String) entry.getValue());
+		return metadata;
+	}
+
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge;
+
+//JDK imports
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Vector;
+import java.util.logging.FileHandler;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.logging.SimpleFormatter;
+import java.util.regex.Pattern;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+
+//OODT imports
+import org.apache.oodt.cas.crawl.ProductCrawler;
+import org.apache.oodt.cas.crawl.StdProductCrawler;
+import org.apache.oodt.cas.crawl.status.IngestStatus;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.oodt.cas.pge.config.DynamicConfigFile;
+import org.apache.oodt.cas.pge.config.FileStagingInfo;
+import org.apache.oodt.cas.pge.config.OutputDir;
+import org.apache.oodt.cas.pge.config.PgeConfig;
+import org.apache.oodt.cas.pge.config.PgeConfigBuilder;
+import org.apache.oodt.cas.pge.config.RegExprOutputFiles;
+import org.apache.oodt.cas.pge.config.RenamingConv;
+import org.apache.oodt.cas.pge.config.XmlFilePgeConfigBuilder;
+import org.apache.oodt.cas.pge.metadata.PGETaskMetKeys;
+import org.apache.oodt.cas.pge.staging.FileManagerFileStager;
+import org.apache.oodt.cas.pge.staging.FileStager;
+import org.apache.oodt.cas.pge.writers.PcsMetFileWriter;
+import org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter;
+import org.apache.oodt.commons.exec.ExecUtils;
+import org.apache.oodt.commons.xml.XMLUtils;
+
+//JPL OODT imports
+import org.apache.oodt.cas.workflow.instance.TaskInstance;
+import org.apache.oodt.cas.workflow.metadata.ControlMetadata;
+import org.apache.oodt.cas.workflow.state.results.ResultsFailureState;
+import org.apache.oodt.cas.workflow.state.results.ResultsState;
+import org.apache.oodt.cas.workflow.state.results.ResultsSuccessState;
+import org.apache.oodt.cas.workflow.util.ScriptFile;
+import org.apache.oodt.cas.filemgr.ingest.StdIngester;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+
+//Spring imports
+import org.springframework.context.support.FileSystemXmlApplicationContext;
+
+
+/**
+ * 
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * Runs a CAS-style Product Generation Executive based on the PCS Wrapper
+ * Architecture from mattmann et al. on OCO
+ * </p>.
+ */
+public class PGETaskInstance extends TaskInstance {
+    
+    protected static final Logger LOG = Logger.getLogger(PGETaskInstance.class.getName());
+        
+    protected PGETaskInstance() {
+    	super();
+    }
+    
+    protected PgeConfig createPgeConfig(ControlMetadata ctrlMetadata) throws Exception {
+    	String pgeConfigBuilderClass = ctrlMetadata.getMetadata(PGETaskMetKeys.PGE_CONFIG_BUILDER);
+    	if (pgeConfigBuilderClass != null)
+    		return ((PgeConfigBuilder) Class.forName(pgeConfigBuilderClass).newInstance()).build(ctrlMetadata);
+    	else
+    		return new XmlFilePgeConfigBuilder().build(ctrlMetadata);
+    }
+    
+    protected void runPropertyAdders(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+        try {
+            // load/run property adders
+            String propertyAdders = ctrlMetadata.getMetadata(PGETaskMetKeys.PROPERTY_ADDERS);
+            if (propertyAdders != null)
+            	for (String propertyAdder : propertyAdders.split(","))
+            		this.runPropertyAdder(this.loadPropertyAdder(propertyAdder, pgeConfig, ctrlMetadata), pgeConfig, ctrlMetadata);
+        } catch (Exception e) {
+            throw new Exception("Failed to instanciate/run Property Adders : " + e.getMessage(), e);
+        }
+    }
+
+    protected ConfigFilePropertyAdder loadPropertyAdder(
+            String propertyAdderClasspath, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+        return (ConfigFilePropertyAdder) Class.forName(propertyAdderClasspath)
+                .newInstance();
+    }
+
+    protected void runPropertyAdder(ConfigFilePropertyAdder propAdder, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        propAdder.addConfigProperties(ctrlMetadata, pgeConfig
+                .getPropertyAdderCustomArgs());
+    }
+
+    protected void createExeDir(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        LOG.log(Level.INFO, "Creating PGE execution working directory: ["
+                + pgeConfig.getExeDir() + "]");
+        new File(pgeConfig.getExeDir()).mkdirs();
+    }
+
+    protected void createOuputDirsIfRequested(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        for (OutputDir outputDir : pgeConfig.getOuputDirs()) {
+            if (outputDir.isCreateBeforeExe()) {
+                LOG.log(Level.INFO, "Creating PGE file ouput directory: ["
+                        + outputDir.getPath() + "]");
+                new File(outputDir.getPath()).mkdirs();
+            }
+        }
+    }
+
+    protected void ensureFilesStaged(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+    	if (pgeConfig.getFileStagingInfo() != null) {
+    		FileStager fileStager = this.getFileStager(pgeConfig, ctrlMetadata);
+    		if (fileStager != null)
+    			fileStager.stageFiles(pgeConfig.getFileStagingInfo(), ctrlMetadata);
+    	}
+    }
+    
+    protected FileStager getFileStager(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+    	return new FileManagerFileStager();
+    }
+    
+    protected void createSciPgeConfigFiles(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+//        this.update(new ExecutingState("Building Config Files"), ctrlMetadata);
+        for (DynamicConfigFile dynamicConfigFile : pgeConfig
+                .getDynamicConfigFiles()) {
+            try {
+                this.createSciPgeConfigFile(dynamicConfigFile, pgeConfig, ctrlMetadata);
+            } catch (Exception e) {
+                e.printStackTrace();
+                throw new IOException(
+                        "Failed to created pge input config file ' "
+                                + dynamicConfigFile.getFilePath() + "' : "
+                                + e.getMessage());
+            }
+        }
+    }
+
+    protected void createSciPgeConfigFile(DynamicConfigFile dynamicConfigFile, PgeConfig pgeConfig, ControlMetadata ctrlMetadata)
+            throws Exception {
+        File parentDir = new File(dynamicConfigFile.getFilePath())
+                .getParentFile();
+        if (!parentDir.exists())
+            parentDir.mkdirs();
+        SciPgeConfigFileWriter writer = (SciPgeConfigFileWriter) Class.forName(
+                dynamicConfigFile.getWriterClass()).newInstance();
+        writer.createConfigFile(dynamicConfigFile.getFilePath(),
+        		ctrlMetadata.asMetadata(), dynamicConfigFile.getArgs());
+    }
+
+    protected void processOutput(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws FileNotFoundException, IOException {
+        for (final OutputDir outputDir : pgeConfig.getOuputDirs()) {
+            File[] createdFiles = new File(outputDir.getPath()).listFiles();
+            for (File createdFile : createdFiles) {
+                Metadata outputMetadata = new Metadata();
+                for (RegExprOutputFiles regExprFiles : outputDir
+                        .getRegExprOutputFiles()) {
+                    if (Pattern.matches(regExprFiles.getRegExp(), createdFile
+                            .getName())) {
+                        try {
+                            PcsMetFileWriter writer = (PcsMetFileWriter) Class
+                            	.forName(regExprFiles.getConverterClass())
+                            	.newInstance();
+                            outputMetadata.replaceMetadata(this.getMetadataForFile(
+						        (regExprFiles.getRenamingConv() != null) 
+							    ? createdFile = this.renameFile(createdFile, regExprFiles.getRenamingConv(), pgeConfig, ctrlMetadata)
+							    : createdFile, writer, regExprFiles.getArgs(), pgeConfig, ctrlMetadata));
+                        } catch (Exception e) {
+                            LOG.log(Level.SEVERE,
+                                    "Failed to create metadata file for '"
+                                            + createdFile + "' : "
+                                            + e.getMessage(), e);
+                        }
+                    }
+                }
+                if (outputMetadata.getAllKeys().size() > 0)
+                	this.writeMetadataToFile(outputMetadata, createdFile.getAbsolutePath() 
+                			+ "." + ctrlMetadata.getMetadata(PGETaskMetKeys.MET_FILE_EXT));
+            }
+        }
+    }
+    
+    protected File renameFile(File file, RenamingConv renamingConv, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+    	Metadata curMetadata = ctrlMetadata.asMetadata();
+    	curMetadata.replaceMetadata(renamingConv.getTmpReplaceMet());
+    	String newFileName = PathUtils.doDynamicReplacement(renamingConv.getRenamingString(), curMetadata);
+    	File newFile = new File(file.getParentFile(), newFileName);
+        LOG.log(Level.INFO, "Renaming file '" + file.getAbsolutePath() + "' to '" + newFile.getAbsolutePath() + "'");
+    	if (!file.renameTo(newFile))
+    		throw new IOException("Renaming returned false");
+    	return newFile;
+    }
+
+    protected Metadata getMetadataForFile(File sciPgeCreatedDataFile,
+            PcsMetFileWriter writer, Object[] args, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+        return writer.getMetadataForFile(sciPgeCreatedDataFile, ctrlMetadata, args);
+    }
+    
+    protected void writeMetadataToFile(Metadata metadata, String toMetFilePath) 
+    		throws FileNotFoundException, IOException {
+		new SerializableMetadata(metadata, "UTF-8", false)
+				.writeMetadataToXmlStream(new FileOutputStream(toMetFilePath));
+	}
+    
+    protected ScriptFile buildPgeRunScript(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        ScriptFile sf = new ScriptFile(pgeConfig.getShellType());
+        sf.setCommands(pgeConfig.getExeCmds());
+        return sf;
+    }
+
+    protected String getScriptPath(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        return new File(pgeConfig.getExeDir()).getAbsolutePath() + "/"
+                + this.getPgeScriptName(pgeConfig, ctrlMetadata);
+    }
+
+    protected String getPgeScriptName(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        return "sciPgeExeScript_" + ctrlMetadata.getMetadata(PGETaskMetKeys.NAME);
+    }
+
+    protected String getDumpMetadataPath(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        return new File(pgeConfig.getExeDir()).getAbsolutePath() + "/"
+                + this.getDumpMetadataName(pgeConfig, ctrlMetadata);
+    }
+    
+    protected String getDumpMetadataName(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        return "pgetask-metadata.xml";
+    }
+
+    protected Handler initializePgeLogHandler(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws SecurityException,
+			IOException {
+    	FileHandler handler = null;
+		String logFilePattern = ctrlMetadata.getMetadata(PGETaskMetKeys.LOG_FILE_PATTERN);
+		if (logFilePattern != null) {
+			LOG.log(Level.INFO, "Creating Log Handler to capture pge output to file '"
+							+ logFilePattern + "'");
+			new File(logFilePattern).getParentFile().mkdirs();
+			handler = new FileHandler(logFilePattern);
+			handler.setFormatter(new SimpleFormatter());
+
+		}
+		return handler;
+	}
+    
+    protected Logger initializePgeLogger(Handler handler, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+    	if (handler != null) {
+	    	Logger pgeLogger = Logger.getLogger(ctrlMetadata
+					.getMetadata(PGETaskMetKeys.NAME)
+					+ System.currentTimeMillis());
+			pgeLogger.addHandler(handler);
+			return pgeLogger;
+    	}else {
+    		return LOG;
+    	}
+    }
+    
+    protected void closePgeLogHandler(Logger logger, Handler handler, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+    	if (logger != null && handler != null) {
+    		logger.removeHandler(handler);
+    		handler.close();
+    	}
+    }
+    
+    protected void runPge(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+        ScriptFile sf = null;
+        Handler handler = null;
+        Logger pgeLogger = null;
+        try {
+            long startTime = System.currentTimeMillis();
+
+            // create script to run
+            sf = this.buildPgeRunScript(pgeConfig, ctrlMetadata);
+            sf.writeScriptFile(this.getScriptPath(pgeConfig, ctrlMetadata));
+
+            // run script and evaluate whether success or failure
+            handler = this.initializePgeLogHandler(pgeConfig, ctrlMetadata);
+            pgeLogger = this.initializePgeLogger(handler, pgeConfig, ctrlMetadata);
+//            this.update(new ExecutingState("Running PGE"), ctrlMetadata);
+            if (!this.wasPgeSuccessful(ExecUtils.callProgram(pgeConfig
+                    .getShellType()
+                    + " " + this.getScriptPath(pgeConfig, ctrlMetadata), pgeLogger, new File(pgeConfig
+                    .getExeDir()).getAbsoluteFile()), pgeConfig, ctrlMetadata))
+                throw new RuntimeException("Pge didn't finish successfully");
+            else
+                LOG.log(Level.INFO, "Successfully completed running: '"
+                        + sf.getCommands() + "'");
+           
+            
+            long endTime = System.currentTimeMillis();
+            ctrlMetadata.replaceLocalMetadata(PGETaskMetKeys.PGE_RUNTIME,
+                    (endTime - startTime) + "");
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new Exception("Exception when executing PGE commands '"
+                    + (sf != null ? sf.getCommands() : "NULL") + "' : "
+                    + e.getMessage());
+        }finally {
+        	this.closePgeLogHandler(pgeLogger, handler, pgeConfig, ctrlMetadata);
+        }
+    }
+
+    protected boolean wasPgeSuccessful(int returnCode, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        return returnCode == 0;
+    }
+
+    protected void ingestProducts(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+        ProductCrawler crawler = this.createProductCrawler(pgeConfig, ctrlMetadata); 
+        this.configureCrawler(crawler, pgeConfig, ctrlMetadata);
+        this.runIngestCrawler(crawler, this.getOutputDirs(pgeConfig, ctrlMetadata), pgeConfig, ctrlMetadata);
+    }
+    
+    protected ProductCrawler createProductCrawler(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+    	StdProductCrawler crawler = new StdProductCrawler();
+        crawler.setMetFileExtension(ctrlMetadata
+                .getMetadata(PGETaskMetKeys.MET_FILE_EXT));
+    	return crawler;
+    }
+
+    protected List<File> getOutputDirs(PgeConfig pgeConfig, ControlMetadata ctrlMetadata) {
+        List<File> outputDirs = new LinkedList<File>();
+        for (OutputDir outputDir : pgeConfig.getOuputDirs())
+            outputDirs.add(new File(outputDir.getPath()));
+        return outputDirs;
+    }
+
+    protected void configureCrawler(ProductCrawler crawler, PgeConfig pgeConfig, ControlMetadata ctrlMetadata)
+            throws Exception {
+    	crawler.setClientTransferer(ctrlMetadata
+                .getMetadata(PGETaskMetKeys.INGEST_CLIENT_TRANSFER_SERVICE_FACTORY));
+        crawler.setFilemgrUrl(ctrlMetadata
+                .getMetadata(PGETaskMetKeys.INGEST_FILE_MANAGER_URL));
+        String actionRepoFile = ctrlMetadata
+                .getMetadata(PGETaskMetKeys.ACTION_REPO_FILE);
+        if (actionRepoFile != null && !actionRepoFile.equals("")) {
+            crawler.setApplicationContext(new FileSystemXmlApplicationContext(
+                    actionRepoFile));
+            crawler.setActionIds(ctrlMetadata
+                    .getAllMetadata(PGETaskMetKeys.ACTION_IDS));
+        }
+        crawler.setRequiredMetadata(ctrlMetadata
+                .getAllMetadata(PGETaskMetKeys.REQUIRED_METADATA));
+        String crawlForDirsString = ctrlMetadata
+                .getMetadata(PGETaskMetKeys.CRAWLER_CRAWL_FOR_DIRS);
+        boolean crawlForDirs = (crawlForDirsString != null) ? crawlForDirsString
+                .toLowerCase().equals("true")
+                : false;
+        String recurString = ctrlMetadata
+                .getMetadata(PGETaskMetKeys.CRAWLER_RECUR);
+        boolean recur = (recurString != null) ? recurString.toLowerCase()
+                .equals("true") : true;
+        crawler.setCrawlForDirs(crawlForDirs);
+        crawler.setNoRecur(!recur);
+    	LOG.log(Level.INFO, "Passing Workflow Metadata to CAS-Crawler as global metadata . . .");
+    	crawler.setGlobalMetadata(ctrlMetadata.asMetadata());
+    }
+
+    protected void runIngestCrawler(ProductCrawler crawler,
+            List<File> crawlDirs, PgeConfig pgeConfig, ControlMetadata ctrlMetadata) throws Exception {
+    	File currentDir = null;
+        try {
+//            this.update(new ExecutingState("Crawling/Ingesting Products"), ctrlMetadata);
+        	boolean attemptIngestAll = Boolean.parseBoolean(ctrlMetadata.getMetadata(PGETaskMetKeys.ATTEMPT_INGEST_ALL));
+            for (File crawlDir : crawlDirs) {
+            	currentDir = crawlDir;
+                LOG.log(Level.INFO,
+                        "Executing StdProductCrawler in productPath: ["
+                                + crawlDir + "]");
+                crawler.crawl(crawlDir);
+                if (!attemptIngestAll)
+                	this.verifyIngests(crawler);
+            }
+            if (attemptIngestAll)
+            	this.verifyIngests(crawler);
+        } catch (Exception e) {
+            throw new Exception(
+                    "Failed while attempting to ingest products while crawling directory '" + currentDir + "' (all products may not have been ingested) : "
+                            + e.getMessage(), e);
+        }
+    }
+    
+    protected void verifyIngests(ProductCrawler crawler) throws Exception {
+    	boolean ingestsSuccess = true;
+    	String exceptionMsg = "";
+    	for (IngestStatus status : crawler.getIngestStatus()) {
+    		if (status.getResult().equals(IngestStatus.Result.FAILURE)) {
+    			exceptionMsg += (exceptionMsg.equals("") ? "" : " : ")
+						+ "Failed to ingest product [file='"
+						+ status.getProduct().getAbsolutePath() + "',result='"
+						+ status.getResult() + "',msg='" + status.getMessage()
+						+ "']";
+    			ingestsSuccess = false;
+    		}else if (!status.getResult().equals(IngestStatus.Result.SUCCESS)) {
+                LOG.log(Level.WARNING, "Product was not ingested [file='"
+						+ status.getProduct().getAbsolutePath() + "',result='"
+						+ status.getResult() + "',msg='" + status.getMessage()
+						+ "']");
+    		}
+    	}
+    	if (!ingestsSuccess)
+    		throw new Exception(exceptionMsg);
+    }
+
+	@Override
+	protected ResultsState performExecution(ControlMetadata ctrlMetadata) {
+        try {
+        	//Initialization
+        	PgeConfig pgeConfig = this.createPgeConfig(ctrlMetadata);
+        	this.runPropertyAdders(pgeConfig, ctrlMetadata);
+        	
+        	//PGE Setup
+            this.createExeDir(pgeConfig, ctrlMetadata);
+            this.createOuputDirsIfRequested(pgeConfig, ctrlMetadata);
+            if (Boolean.parseBoolean(ctrlMetadata.getMetadata(PGETaskMetKeys.DUMP_METADATA))) 
+	        	new SerializableMetadata(ctrlMetadata.asMetadata())
+						.writeMetadataToXmlStream(new FileOutputStream(this
+								.getDumpMetadataPath(pgeConfig, ctrlMetadata)));
+            this.ensureFilesStaged(pgeConfig, ctrlMetadata);
+            this.createSciPgeConfigFiles(pgeConfig, ctrlMetadata);
+            
+            //Run/Process PGE
+            this.runPge(pgeConfig, ctrlMetadata);
+            this.processOutput(pgeConfig, ctrlMetadata);
+            this.ingestProducts(pgeConfig, ctrlMetadata);
+        	return new ResultsSuccessState("Successfully executed PGE");
+        } catch (Exception e) {
+        	LOG.log(Level.SEVERE, "Failed to execute PGE : " + e.getMessage(), e);	
+        	return new ResultsFailureState("Failed to execute PGE : " + e.getMessage());
+        }
+	}
+
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/StdPGETaskInstance.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/StdPGETaskInstance.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/StdPGETaskInstance.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/StdPGETaskInstance.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,19 @@
+//Copyright (c) 2008, California Institute of Technology.
+//ALL RIGHTS RESERVED. U.S. Government sponsorship acknowledged.
+//
+//$Id$
+
+package org.apache.oodt.cas.pge;
+
+/**
+ * 
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * The Standard implementation of the {@link PGETaskInstance}. Doesn't override
+ * any of the default functionality
+ * </p>.
+ */
+public class StdPGETaskInstance extends PGETaskInstance {
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/PGETaskWorkflowCondition.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/PGETaskWorkflowCondition.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/PGETaskWorkflowCondition.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/PGETaskWorkflowCondition.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.condition;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.oodt.cas.pge.metadata.PGETaskMetKeys;
+import org.apache.oodt.commons.date.DateUtils;
+import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery;
+import org.apache.oodt.cas.filemgr.structs.query.QueryFilter;
+import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
+import org.apache.oodt.cas.filemgr.structs.query.conv.VersionConverter;
+import org.apache.oodt.cas.filemgr.structs.query.filter.FilterAlgor;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.util.QueryUtils;
+import org.apache.oodt.cas.filemgr.util.SqlParser;
+import org.apache.oodt.cas.workflow.instance.TaskInstance;
+import org.apache.oodt.cas.workflow.metadata.ControlMetadata;
+import org.apache.oodt.cas.workflow.processor.ProcessorInfo;
+import org.apache.oodt.cas.workflow.state.results.ResultsBailState;
+import org.apache.oodt.cas.workflow.state.results.ResultsFailureState;
+import org.apache.oodt.cas.workflow.state.results.ResultsState;
+import org.apache.oodt.cas.workflow.state.results.ResultsSuccessState;
+
+//JDK imports
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * 
+ * @author bfoster
+ * @version $Revision$
+ * 
+ */
+public class PGETaskWorkflowCondition extends TaskInstance implements PGETaskMetKeys {
+    
+    private static Logger LOG = Logger.getLogger(PGETaskWorkflowCondition.class.getName());
+    
+    public static enum QueryType { START, END, START_INCL, END_INCL, TERM };
+    
+	public static final String ALL_PRODUCT_TYPES_GROUP = "AllProductTypes";
+	
+	public static final String ALL_PGES_GROUP = "AllPGEs";
+	
+	@Override
+	protected ResultsState performExecution(ControlMetadata ctrlMetadata) {
+        try {        	
+        	Metadata metadata = ctrlMetadata.asMetadata();
+        	
+        	String filemgrUrl = PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(QUERY_FILE_MANAGER_URL), metadata);
+
+            // query metadata keys
+            String filterAlgor = ctrlMetadata.getMetadata(FILTER_ALGOR) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(FILTER_ALGOR), metadata) : null;
+            String sortByKey = ctrlMetadata.getMetadata(SORY_BY_KEY) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(SORY_BY_KEY), metadata) : null;
+            String epsilon = ctrlMetadata.getMetadata(EPSILON_IN_MILLIS) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(EPSILON_IN_MILLIS), metadata) : null;
+            String versionConverter = ctrlMetadata.getMetadata(VERSION_CONVERTER) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(VERSION_CONVERTER), metadata) : null;
+            String productTypes = ctrlMetadata.getMetadata(PRODUCT_TYPES) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(PRODUCT_TYPES), metadata) : null;
+            String resultKeyFormats = ctrlMetadata.getMetadata(RESULT_KEY_FORMATS) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(RESULT_KEY_FORMATS), metadata) : null;
+        	String startDateTimeKey = ctrlMetadata.getMetadata(START_DATE_TIME_KEY) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(START_DATE_TIME_KEY), metadata) : null;
+        	String endDateTimeKey = ctrlMetadata.getMetadata(END_DATE_TIME_KEY) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(END_DATE_TIME_KEY), metadata) : null;
+            String sqlQueryKey = ctrlMetadata.getMetadata(SQL_QUERY_KEY) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(SQL_QUERY_KEY), metadata) : null;
+            String versioningKey = ctrlMetadata.getMetadata(VERSIONING_KEY) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(VERSIONING_KEY), metadata) : null;
+
+            // load metadata from property adders
+//            this.update(new ExecutingState("Loading Condition Properties Adders"), ctrlMetadata);
+            String workflowConditionPropAddersString = ctrlMetadata.getMetadata(PROP_ADDERS) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(PROP_ADDERS), metadata) : null;
+            if (workflowConditionPropAddersString != null) {
+	            String[] workflowConditionPropAdders = workflowConditionPropAddersString.split(",");
+	            for (String workflowConditionPropAdder : workflowConditionPropAdders) {
+	            	try {
+	            		workflowConditionPropAdder = workflowConditionPropAdder.trim();
+	            		LOG.log(Level.INFO, "Running WorkflowConditionPropAdder '" + workflowConditionPropAdder + "'");
+	            		((WorkflowConditionPropAdder) Class.forName(workflowConditionPropAdder).newInstance()).addMetadata(ctrlMetadata);
+	            	}catch (Exception e) {
+	            		LOG.log(Level.WARNING, "Failed to run WorkflowConditionPropAdder '" + workflowConditionPropAdder + "' : " + e.getMessage(), e);
+	                    return new ResultsBailState("Precondition Bailed : Failed to run WorkflowConditionPropAdder '" + workflowConditionPropAdder + "' : " + e.getMessage());
+	            	}
+	            }
+            }
+                       
+            // create SQL query
+//            this.update(new ExecutingState("Building Query"), ctrlMetadata);
+            ComplexQuery complexQuery = new ComplexQuery();
+            complexQuery.setReducedProductTypeNames(Collections.unmodifiableList(Arrays.asList(productTypes.split(","))));
+            if (sortByKey != null)
+            	complexQuery.setSortByMetKey(sortByKey);
+            
+            if (filterAlgor != null) {
+            	complexQuery.setQueryFilter(new QueryFilter(startDateTimeKey, endDateTimeKey, versioningKey, (FilterAlgor) Class.forName(filterAlgor).newInstance()));
+            	if (epsilon != null)
+            		complexQuery.getQueryFilter().getFilterAlgor().setEpsilon(Integer.parseInt(epsilon));
+            	if (versionConverter != null)
+            		complexQuery.getQueryFilter().setConverter((VersionConverter) Class.forName(versionConverter).newInstance());
+            }
+            
+            List<QueryCriteria> criteria = new LinkedList<QueryCriteria>();
+            List<String> productTypeNames = new Vector<String>(complexQuery.getReducedProductTypeNames());
+            productTypeNames.add("AllProductTypes");
+            HashSet<String> queriedOnElementNames = new HashSet<String>();
+            for (String productType : productTypeNames) {
+            	for (String key : ctrlMetadata.asMetadata().getAllKeys()) {
+            		if (key.startsWith(productType + "/")) {
+            			String[] splitMetKey = key.split("\\/");
+            			String elementName = splitMetKey[1];
+            			String queryTypeString = splitMetKey[2];
+            			String elementNameAndQueryType = elementName + "/" + queryTypeString;
+            			if (!queriedOnElementNames.contains(elementNameAndQueryType) && queryTypeString != null) {
+	            			QueryType queryType = QueryType.valueOf(queryTypeString.toUpperCase());
+	            			if (queryType.equals(QueryType.START_INCL)) {
+			            		RangeQueryCriteria startRangeQC = new RangeQueryCriteria();
+			            		startRangeQC.setElementName(elementName);
+			            		startRangeQC.setStartValue(ctrlMetadata.getMetadata(key));
+			            		startRangeQC.setEndValue(null);
+			            		startRangeQC.setInclusive(true);
+			            		criteria.add(startRangeQC);
+			            		queriedOnElementNames.add(elementNameAndQueryType);
+	            			}else if (queryType.equals(QueryType.START)) {
+			            		RangeQueryCriteria startRangeQC = new RangeQueryCriteria();
+			            		startRangeQC.setElementName(elementName);
+			            		startRangeQC.setStartValue(ctrlMetadata.getMetadata(key));
+			            		startRangeQC.setEndValue(null);
+			            		startRangeQC.setInclusive(false);
+			            		criteria.add(startRangeQC);
+			            		queriedOnElementNames.add(elementNameAndQueryType);
+	            			}else if (queryType.equals(QueryType.END_INCL)) {
+			            		RangeQueryCriteria endRangeQC = new RangeQueryCriteria();
+			            		endRangeQC.setElementName(elementName);
+			            		endRangeQC.setStartValue(null);
+			            		endRangeQC.setEndValue(ctrlMetadata.getMetadata(key));
+			            		endRangeQC.setInclusive(true);
+			            		criteria.add(endRangeQC);
+			            		queriedOnElementNames.add(elementNameAndQueryType);
+	            			}else if (queryType.equals(QueryType.END)) {
+			            		RangeQueryCriteria endRangeQC = new RangeQueryCriteria();
+			            		endRangeQC.setElementName(elementName);
+			            		endRangeQC.setStartValue(null);
+			            		endRangeQC.setEndValue(ctrlMetadata.getMetadata(key));
+			            		endRangeQC.setInclusive(false);
+			            		criteria.add(endRangeQC);
+			            		queriedOnElementNames.add(elementNameAndQueryType);
+	            			}else if (queryType.equals(QueryType.TERM)) {
+								TermQueryCriteria termQC = new TermQueryCriteria();
+								termQC.setElementName(elementName);
+								termQC.setValue(ctrlMetadata.getMetadata(key));
+								criteria.add(termQC);
+			            		queriedOnElementNames.add(elementNameAndQueryType);
+	            			}
+            			}
+            		} 
+            			
+            	}
+            }
+            metadata = ctrlMetadata.asMetadata();
+            
+            // add all QueryCriteria to final query
+            complexQuery.setCriteria(criteria);
+
+            int minNumOfFiles = 1;
+            String minNumOfFilesString = ctrlMetadata.getMetadata(MIN_NUM_OF_FILES) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(MIN_NUM_OF_FILES), metadata) : null;
+            if (minNumOfFilesString != null)
+                minNumOfFiles = Integer.parseInt(minNumOfFilesString);
+            
+            long maxGapSize = -1;
+            String maxGapString = ctrlMetadata.getMetadata(MAX_GAP_SIZE) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(MAX_GAP_SIZE), metadata) : null;
+            if (maxGapString != null) 
+            	maxGapSize = Long.parseLong(maxGapString);
+            String maxGapStartDateTime = ctrlMetadata.getMetadata(MAX_GAP_START_DATE_TIME) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(MAX_GAP_START_DATE_TIME), metadata) : null;
+            String maxGapEndDateTime = ctrlMetadata.getMetadata(MAX_GAP_END_DATE_TIME) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(MAX_GAP_END_DATE_TIME), metadata) : null;
+            
+            String timeoutString = ctrlMetadata.getMetadata(TIMEOUT) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(TIMEOUT), metadata) : null;
+            double timeout = -1;
+            if (timeoutString != null)
+                timeout = Double.parseDouble(timeoutString);
+            
+            int expectedNumOfFiles = -1;
+            String expectedNumOfFilesString = ctrlMetadata.getMetadata(EXPECTED_NUM_OF_FILES) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(EXPECTED_NUM_OF_FILES), metadata) : null;
+            if (expectedNumOfFilesString != null)
+                expectedNumOfFiles = Integer.parseInt(expectedNumOfFilesString);
+            else
+            	return new ResultsFailureState("Precondition Failed : Must set metadata field '" + EXPECTED_NUM_OF_FILES + "'");
+                        
+//            this.update(new ExecutingState("Performing query '" + SqlParser.unparseSqlQuery(complexQuery) + "'"), ctrlMetadata);
+            List<QueryResult> results = new XmlRpcFileManagerClient(new URL(filemgrUrl)).complexQuery(complexQuery);
+            
+            // Add Results to Metadata
+            if (resultKeyFormats != null) {
+	        	Pattern pattern = Pattern.compile("\\{.*?\\|.*?\\}");
+	        	Matcher matcher = pattern.matcher(resultKeyFormats);
+	        	while(matcher.find()) {
+	        		String keyFormatSet = resultKeyFormats.substring(matcher.start() + 1, matcher.end() - 1);
+	        		String key = keyFormatSet.substring(0, keyFormatSet.indexOf("|")).trim();
+	        		String format = keyFormatSet.substring(keyFormatSet.indexOf("|") + 1).trim();
+	        		ctrlMetadata.replaceLocalMetadata(key, Arrays.asList(QueryUtils.getQueryResultsAsFormattedString(results, format, ",").split(",")));
+	        		ctrlMetadata.setAsWorkflowMetadataKey(key);
+	        	}
+            }
+            if (sqlQueryKey != null) {
+            	ctrlMetadata.replaceLocalMetadata(sqlQueryKey, SqlParser.unparseSqlQuery(complexQuery));
+            	ctrlMetadata.setAsWorkflowMetadataKey(sqlQueryKey);
+            }
+            
+            ctrlMetadata.commitWorkflowMetadataKeys();
+            metadata = ctrlMetadata.asMetadata();
+            
+            // Run post-property-adders
+            String postWorkflowConditionPropAddersString = ctrlMetadata.getMetadata(POST_PROP_ADDERS) != null ? PathUtils.doDynamicReplacement(ctrlMetadata.getMetadata(POST_PROP_ADDERS), metadata) : null;
+            if (postWorkflowConditionPropAddersString != null) {
+	            String[] workflowConditionPropAdders = postWorkflowConditionPropAddersString.split(",");
+	            for (String workflowConditionPropAdder : workflowConditionPropAdders) {
+	            	try {
+	            		workflowConditionPropAdder = workflowConditionPropAdder.trim();
+	            		LOG.log(Level.INFO, "Running Post-WorkflowConditionPropAdder '" + workflowConditionPropAdder + "'");
+	            		((WorkflowConditionPropAdder) Class.forName(workflowConditionPropAdder).newInstance()).addMetadata(ctrlMetadata);
+	            	}catch (Exception e) {
+	            		LOG.log(Level.WARNING, "Failed to run PostWorkflowConditionPropAdder '" + workflowConditionPropAdder + "' : " + e.getMessage(), e);
+	                    return new ResultsBailState("Precondition Bailed : Failed to run PostWorkflowConditionPropAdder '" + workflowConditionPropAdder + "' : " + e.getMessage());
+	            	}
+	            }
+            }
+            
+            ProcessorInfo processorInfo = this.getProcessorInfo();
+            Calendar calendar = Calendar.getInstance();
+            calendar.setTime(processorInfo.getExecutionDate());
+            long startTime = calendar.getTimeInMillis();
+            boolean timeoutHit = (int) timeout == 0 || (timeout != -1 && (System.currentTimeMillis() - startTime) / 1000.0 >= timeout);
+            if (timeoutHit && results.size() < minNumOfFiles)
+            	return new ResultsFailureState("Precondition Failed : Timeout reached, min number of results, " + minNumOfFiles + ", not found : only found '" + results.size() + "' results");
+            if (timeoutHit && results.size() >= minNumOfFiles) {
+                return new ResultsSuccessState("Precondition : Timeout reached min number of files, " + expectedNumOfFiles + ", found : found '" + results.size() + "' results");
+            }else if (results.size() >= expectedNumOfFiles) {
+            	if (maxGapSize != -1) {
+            		long largestGapFound = -1;
+            		if ((largestGapFound = findLargestGap(results, startDateTimeKey, endDateTimeKey, maxGapStartDateTime, maxGapEndDateTime)) <= maxGapSize)
+            			return new ResultsSuccessState("Precondition : Expected number of files, '" + expectedNumOfFiles + ", found : Passed Gap Analysis [Size=" + maxGapSize + ";StartDateTime=" + maxGapStartDateTime + ";EndDateTime=" + maxGapEndDateTime + ";LargestGap=" + largestGapFound + "]");
+            		else
+            			return new ResultsBailState("Precondition : Expected number of files, '" + expectedNumOfFiles + ", found : Failed Gap Analysis [Size=" + maxGapSize + ";StartDateTime=" + maxGapStartDateTime + ";EndDateTime=" + maxGapEndDateTime + ";LargestGap=" + largestGapFound + "]");
+            	}else {
+            		return new ResultsSuccessState("Precondition Bailed : Expected number of results, " + expectedNumOfFiles + ", found : Gap Analysis turned off");
+            	}
+            }else {
+        		return new ResultsBailState("Precondition Bailed : Expected number of results, " + expectedNumOfFiles + " not found : only found '" + results.size() + "' results");
+            }
+            
+        }catch (Exception e) {
+            LOG.log(Level.SEVERE, "Precondition Failed : " + e.getMessage(), e);
+            return new ResultsBailState("Precondition Bailed : " + e.getMessage());
+        }
+    }
+    
+    private static long findLargestGap(List<QueryResult> results, final String startDateTimeKey, final String endDateTimeKey, String startDateTimeRange, String endDateTimeRange) throws Exception {
+    	List<QueryResult> timeSortedList = new Vector<QueryResult>(results);
+    	Collections.sort(timeSortedList, new Comparator<QueryResult>() {
+			public int compare(QueryResult o1, QueryResult o2) {
+				try {
+					long firstStartDateTime = DateUtils.toCalendar(o1.getMetadata().getMetadata(startDateTimeKey), DateUtils.FormatType.UTC_FORMAT).getTimeInMillis();
+					long secondStartDateTime = DateUtils.toCalendar(o2.getMetadata().getMetadata(startDateTimeKey), DateUtils.FormatType.UTC_FORMAT).getTimeInMillis();
+					return new Long(firstStartDateTime).compareTo(secondStartDateTime);
+				}catch (Exception e) {
+					LOG.log(Level.SEVERE, "Error occurred during results time sort for gap analysis : " + e.getMessage(), e);
+					return 0;
+				}
+			}
+    	});
+    	if (startDateTimeKey == null || endDateTimeKey == null)
+    		throw new Exception("You must specify the metadata fields '" + START_DATE_TIME_KEY + "' and '" + END_DATE_TIME_KEY + "' to use gap analysis");
+    	if (startDateTimeRange == null)
+    		LOG.log(Level.WARNING, "You should specify the metadata field '" + MAX_GAP_START_DATE_TIME + "' if you want gap analysis to be tied to a start time range");
+    	if (endDateTimeRange == null)
+    		LOG.log(Level.WARNING, "You should specify the metadata field '" + MAX_GAP_END_DATE_TIME + "' if you want gap analysis to be tied to an end time range");
+    	
+        if (timeSortedList.size() > 0) {
+            long largestGapSize = 0;
+            if (startDateTimeRange != null) {
+            	largestGapSize = DateUtils.toCalendar(timeSortedList.get(0).getMetadata().getMetadata(startDateTimeKey), DateUtils.FormatType.UTC_FORMAT).getTimeInMillis() 
+            					- DateUtils.toCalendar(startDateTimeRange, DateUtils.FormatType.UTC_FORMAT).getTimeInMillis();
+            }
+            for (int i = 1; i < timeSortedList.size(); i++) {
+                long previousEventEndTime = DateUtils.toCalendar(
+                		timeSortedList.get(i-1).getMetadata().getMetadata(endDateTimeKey), 
+                        DateUtils.FormatType.UTC_FORMAT).getTimeInMillis();
+                long currentEventStartTime = DateUtils.toCalendar(
+                		timeSortedList.get(i).getMetadata().getMetadata(startDateTimeKey), 
+                        DateUtils.FormatType.UTC_FORMAT).getTimeInMillis();
+                long curGapSize = currentEventStartTime - previousEventEndTime;
+                if (curGapSize > largestGapSize)
+                	largestGapSize = curGapSize;
+            }
+            if (endDateTimeRange != null) {
+            	long lastGapSize = DateUtils.toCalendar(endDateTimeRange, DateUtils.FormatType.UTC_FORMAT).getTimeInMillis() 
+            					- DateUtils.toCalendar(timeSortedList.get(timeSortedList.size() - 1).getMetadata().getMetadata(endDateTimeKey), DateUtils.FormatType.UTC_FORMAT).getTimeInMillis();
+            	if (lastGapSize > largestGapSize)
+            		largestGapSize = lastGapSize;
+            }
+            return largestGapSize;
+        }else {
+            return 0;
+        }
+    }
+    
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/WorkflowConditionPropAdder.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/WorkflowConditionPropAdder.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/WorkflowConditionPropAdder.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/condition/WorkflowConditionPropAdder.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.condition;
+
+//OODT imports
+import org.apache.oodt.cas.workflow.metadata.ControlMetadata;
+
+/**
+ * 
+ * @author bfoster
+ *
+ */
+public interface WorkflowConditionPropAdder {
+
+	public void addMetadata(ControlMetadata ctrlMetadata) throws Exception;
+	
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/DynamicConfigFile.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/DynamicConfigFile.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/DynamicConfigFile.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/DynamicConfigFile.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.config;
+
+/**
+ * 
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * A dynamic representation of a SciPgeConfigFile
+ * </p>.
+ */
+public class DynamicConfigFile {
+
+    private String filePath, writerClass;
+
+    private Object[] args;
+
+    public DynamicConfigFile(String filePath, String writerClass, Object[] args) {
+        this.filePath = filePath;
+        this.writerClass = writerClass;
+        this.args = args;
+    }
+
+    public void setFilePath(String filePath) {
+        this.filePath = filePath;
+    }
+
+    public String getFilePath() {
+        return this.filePath;
+    }
+
+    public void setWriterClass(String writerClass) {
+        this.writerClass = writerClass;
+    }
+
+    public String getWriterClass() {
+        return this.writerClass;
+    }
+
+    public void setArgs(Object[] args) {
+        this.args = args;
+    }
+
+    public Object[] getArgs() {
+        return this.args;
+    }
+
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileBasedPgeConfigBuilder.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileBasedPgeConfigBuilder.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileBasedPgeConfigBuilder.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileBasedPgeConfigBuilder.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.config;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer;
+import org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory;
+import org.apache.oodt.cas.filemgr.datatransfer.RemoteDataTransferFactory;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
+import org.apache.oodt.cas.pge.metadata.PGETaskMetKeys;
+import org.apache.oodt.cas.workflow.metadata.ControlMetadata;
+
+//JDK imports
+import java.io.File;
+import java.net.URL;
+import java.util.Collections;
+
+//Apache imports
+import org.apache.commons.io.FileUtils;
+
+/**
+ * 
+ * @author bfoster
+ *
+ */
+public abstract class FileBasedPgeConfigBuilder implements PgeConfigBuilder {
+	
+	public static final String FILE_BASED_CONFIG_GROUP = PGETaskMetKeys.PGE_TASK_GROUP + "/FileBasedConfig";
+    public static final String STAGE_CONFIG_FILE = FILE_BASED_CONFIG_GROUP + "/StageConfigFile";
+    public static final String DATA_TRANSERER = FILE_BASED_CONFIG_GROUP + "/DataTransferer";
+	public static final String CONFIG_FILE_PATH = FILE_BASED_CONFIG_GROUP + "/ConfigFilePath";
+	
+	public PgeConfig build(ControlMetadata ctrlMetadata) throws Exception {
+		File stagedConfigFile = stagePgeConfig(ctrlMetadata);
+		if (stagedConfigFile != null) {
+			//update config metadata path
+			ctrlMetadata.replaceLocalMetadata(CONFIG_FILE_PATH, stagedConfigFile.getAbsolutePath());
+			if (ctrlMetadata.getMetadata(CONFIG_FILE_PATH, ControlMetadata.DYN) != null) 
+				ctrlMetadata.commitWorkflowMetadataKeys(CONFIG_FILE_PATH);
+		}
+		
+		PgeConfig pgeConfig = this._build(ctrlMetadata);
+		
+		if (stagedConfigFile != null) {
+			//move config file to exe dir
+			new LocalDataTransferFactory().createDataTransfer().copyProduct(createDummyProduct(stagedConfigFile.getAbsolutePath()), new File(pgeConfig.getExeDir()));
+			
+			//update config metadata path
+			ctrlMetadata.replaceLocalMetadata(CONFIG_FILE_PATH, pgeConfig.getExeDir() + "/" + stagedConfigFile.getName());
+			if (ctrlMetadata.getMetadata(CONFIG_FILE_PATH, ControlMetadata.DYN) != null) 
+				ctrlMetadata.commitWorkflowMetadataKeys(CONFIG_FILE_PATH);
+			
+			FileUtils.deleteDirectory(stagedConfigFile.getParentFile());
+		}
+		
+		return pgeConfig;
+	}
+	
+	protected abstract PgeConfig _build(ControlMetadata ctrlMetadata) throws Exception;
+	
+    private static File stagePgeConfig(ControlMetadata ctrlMetadata) throws Exception {
+    	if (Boolean.parseBoolean(ctrlMetadata.getMetadata(STAGE_CONFIG_FILE))) {
+    		String origPath = ctrlMetadata.getMetadata(CONFIG_FILE_PATH);
+    		File origFile = new File(origPath);
+    		if (!origFile.exists()) {
+	    		String configFileName = origFile.getName();
+	    		
+	    		//ensure tmp dir exists
+	    		File tempFile = File.createTempFile("cas-pge", "bogus");
+	    		tempFile.delete();
+	    		File tmpDir = tempFile.getParentFile();
+	    		File casPgeTmpDir = new File(tmpDir, ".cas-pge/" + System.currentTimeMillis());
+	    		casPgeTmpDir.mkdirs();
+	    				    		
+	    		//stage config file
+				DataTransfer dataTransferer = null;
+				if (ctrlMetadata.getMetadata(DATA_TRANSERER) != null) 
+					dataTransferer = GenericFileManagerObjectFactory.getDataTransferServiceFromFactory(ctrlMetadata.getMetadata(DATA_TRANSERER));
+				else
+					dataTransferer = new RemoteDataTransferFactory().createDataTransfer();
+		   		dataTransferer.setFileManagerUrl(new URL(ctrlMetadata.getMetadata(PGETaskMetKeys.QUERY_FILE_MANAGER_URL)));
+				dataTransferer.copyProduct(createDummyProduct(origPath), casPgeTmpDir);
+				
+				return new File(casPgeTmpDir, configFileName);
+    		}
+    	}
+    	return null;
+    }
+
+	private static Product createDummyProduct(String path) {
+		Product dummy = new Product();
+		Reference reference = new Reference();
+		reference.setDataStoreReference("file:" + new File(path).getAbsolutePath());
+		dummy.setProductReferences(Collections.singletonList(reference));
+		return dummy;
+	}
+	
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.config;
+
+//JDK imports
+import java.util.List;
+import java.util.Vector;
+
+/**
+ * 
+ * @author bfoster
+ *
+ */
+public class FileStagingInfo {
+
+	private String stagingDir;
+	private boolean forceStaging = false;
+	private Vector<String> filePaths;
+	
+	public FileStagingInfo(String stagingDir, boolean forceStaging) {
+		this.stagingDir = stagingDir;
+		this.forceStaging = forceStaging;
+		this.filePaths = new Vector<String>();
+	}
+
+	public void addFilePath(String filePath) {
+		this.filePaths.add(filePath);
+	}
+	
+	public void addFilePaths(List<String> filePaths) {
+		this.filePaths.addAll(filePaths);
+	}
+	
+	public List<String> getFilePaths() {
+		return this.filePaths;
+	}
+	
+	public String getStagingDir() {
+		return this.stagingDir;
+	}
+
+	public boolean isForceStaging() {
+		return this.forceStaging;
+	}
+	
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/OutputDir.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.config;
+
+//JDK imports
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * 
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * An output dir for PGE execution
+ * </p>.
+ */
+public class OutputDir {
+
+    private String path;
+
+    private List<RegExprOutputFiles> regExprOutputFilesList;
+
+    private boolean createBeforeExe;
+
+    public OutputDir(String path, boolean createBeforeExe) {
+        this.path = path;
+        this.createBeforeExe = createBeforeExe;
+        this.regExprOutputFilesList = new LinkedList<RegExprOutputFiles>();
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+
+    public String getPath() {
+        return this.path;
+    }
+
+    public void addRegExprOutputFiles(RegExprOutputFiles regExprOutputFiles) {
+        this.regExprOutputFilesList.add(regExprOutputFiles);
+    }
+
+    public List<RegExprOutputFiles> getRegExprOutputFiles() {
+        return this.regExprOutputFilesList;
+    }
+
+    public void setCreateBeforeExe(boolean createBeforeExe) {
+        this.createBeforeExe = createBeforeExe;
+    }
+
+    public boolean isCreateBeforeExe() {
+        return this.createBeforeExe;
+    }
+
+}

Added: oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfig.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfig.java?rev=1052149&view=auto
==============================================================================
--- oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfig.java (added)
+++ oodt/branches/wengine-branch/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfig.java Thu Dec 23 02:48:35 2010
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.config;
+
+//JDK imports
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * 
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * Configuration file for CAS-PGE
+ * </p>.
+ */
+public class PgeConfig {
+
+    private List<DynamicConfigFile> dynamicConfigFiles;
+
+    private List<OutputDir> outputDirs;
+
+    private Object[] propertyAdderCustomArgs;
+
+    private String exeDir;
+
+    private String shellType;
+
+    private List<String> exeCmds;
+    
+    private FileStagingInfo fileStagingInfo;
+    
+    public PgeConfig() {
+        this.shellType = "sh";
+        this.outputDirs = new LinkedList<OutputDir>();
+        this.dynamicConfigFiles = new LinkedList<DynamicConfigFile>();
+        this.exeCmds = new LinkedList<String>();
+    }
+
+    public void addDynamicConfigFile(DynamicConfigFile dynamicConfigFile) {
+        dynamicConfigFiles.add(dynamicConfigFile);
+    }
+
+    public List<DynamicConfigFile> getDynamicConfigFiles() {
+        return this.dynamicConfigFiles;
+    }
+
+    public void addOuputDirAndExpressions(OutputDir outputDir) {
+        this.outputDirs.add(outputDir);
+    }
+
+    public List<OutputDir> getOuputDirs() {
+        return this.outputDirs;
+    }
+
+    public void setExeDir(String exeDir) {
+        this.exeDir = exeDir;
+    }
+
+    public String getExeDir() {
+        return this.exeDir;
+    }
+
+    public void setShellType(String shellType) {
+        if (shellType != null && !shellType.equals(""))
+            this.shellType = shellType;
+    }
+
+    public String getShellType() {
+        return this.shellType;
+    }
+
+    public void setExeCmds(List<String> exeCmds) {
+        this.exeCmds = exeCmds;
+    }
+
+    public List<String> getExeCmds() {
+        return this.exeCmds;
+    }
+
+    public void setPropertyAdderCustomArgs(Object[] args) {
+        propertyAdderCustomArgs = args;
+    }
+
+    public Object[] getPropertyAdderCustomArgs() {
+        return propertyAdderCustomArgs != null ? propertyAdderCustomArgs
+                : new Object[0];
+    }
+
+    public void setFileStagingInfo(FileStagingInfo fileStagingInfo) {
+    	this.fileStagingInfo = fileStagingInfo;
+    }
+    
+    public FileStagingInfo getFileStagingInfo() {
+    	return this.fileStagingInfo;
+    }
+    
+}



Mime
View raw message