oodt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bfos...@apache.org
Subject svn commit: r1351905 - in /oodt/trunk/pge/src: main/java/org/apache/oodt/cas/pge/config/ main/java/org/apache/oodt/cas/pge/metadata/ main/java/org/apache/oodt/cas/pge/util/ main/java/org/apache/oodt/cas/pge/writers/ main/java/org/apache/oodt/cas/pge/wr...
Date Wed, 20 Jun 2012 00:24:00 GMT
Author: bfoster
Date: Wed Jun 20 00:23:59 2012
New Revision: 1351905

URL: http://svn.apache.org/viewvc?rev=1351905&view=rev
Log:
Add file staging support to CAS-PGE's XmlFilePgeConfigBuilder

---------------
OODT-464

Added:
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/Pair.java   (with props)
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java   (with props)
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java   (with props)
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java   (with props)
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java   (contents, props changed)
      - copied, changed from r1351904, oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java
    oodt/trunk/pge/src/main/testdata/
    oodt/trunk/pge/src/main/testdata/blank-pge-config.xml   (with props)
    oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml   (with props)
    oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml   (with props)
    oodt/trunk/pge/src/main/testdata/data-file-1.txt   (with props)
    oodt/trunk/pge/src/main/testdata/data-file-2.txt   (with props)
    oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml   (with props)
    oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml   (with props)
    oodt/trunk/pge/src/main/testdata/pge-config.xml   (with props)
    oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/config/TestXmlFilePgeConfigBuilder.java   (with props)
    oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/util/TestXmlHelper.java   (with props)
Removed:
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/
Modified:
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/XmlFilePgeConfigBuilder.java
    oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java
    oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/TestPGETaskInstance.java

Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java?rev=1351905&r1=1351904&r2=1351905&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/FileStagingInfo.java Wed Jun 20 00:23:59 2012
@@ -31,10 +31,10 @@ import com.google.common.collect.Sets;
  */
 public class FileStagingInfo {
 
-   private String stagingDir;
-   private boolean forceStaging;
-   private Set<String> filePaths;
-   private Set<String> productIds;
+   private final String stagingDir;
+   private final boolean forceStaging;
+   private final Set<String> filePaths;
+   private final Set<String> productIds;
 
    public FileStagingInfo(String stagingDir) {
       this(stagingDir, false);
@@ -52,7 +52,7 @@ public class FileStagingInfo {
    }
 
    public void addFilePaths(List<String> filePaths) {
-      filePaths.addAll(filePaths);
+      this.filePaths.addAll(filePaths);
    }
 
    public List<String> getFilePaths() {

Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java?rev=1351905&r1=1351904&r2=1351905&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/PgeConfigMetKeys.java Wed Jun 20 00:23:59 2012
@@ -35,9 +35,9 @@ public interface PgeConfigMetKeys {
     public static final String METADATA_TAG = "metadata";
 
     public static final String KEYREF_ATTR = "key-ref";
-    
+
     public static final String KEY_GEN_ATTR = "key-gen";
-    
+
     public static final String KEY_ATTR = "key";
 
     public static final String VAL_ATTR = "val";
@@ -45,11 +45,11 @@ public interface PgeConfigMetKeys {
     public static final String ENV_REPLACE_ATTR = "envReplace";
 
     public static final String ENV_REPLACE_NO_RECUR_ATTR = "envReplace-NoRecur";
-    
+
     public static final String SPLIT_ATTR = "split";
 
     public static final String WORKFLOW_MET_ATTR = "workflowMet";
-    
+
     public static final String DYN_INPUT_FILES_TAG = "dynInputFiles";
 
     public static final String FILE_TAG = "file";
@@ -73,4 +73,12 @@ public interface PgeConfigMetKeys {
     public static final String DIR_TAG = "dir";
 
     public static final String CREATE_BEFORE_EXEC_ATTR = "createBeforeExe";
+
+    public static final String FILE_STAGING_TAG = "fileStaging";
+
+    public static final String STAGE_FILES_TAG = "stageFiles";
+
+    public static final String METADATA_KEY_ATTR = "metadataKey";
+
+    public static final String FORCE_ATTR = "force";
 }

Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/XmlFilePgeConfigBuilder.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/XmlFilePgeConfigBuilder.java?rev=1351905&r1=1351904&r2=1351905&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/XmlFilePgeConfigBuilder.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/config/XmlFilePgeConfigBuilder.java Wed Jun 20 00:23:59 2012
@@ -17,61 +17,42 @@
 package org.apache.oodt.cas.pge.config;
 
 //OODT static imports
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.ARGS_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.CMD_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.CREATE_BEFORE_EXEC_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.CUSTOM_METADATA_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.DIR_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.DIR_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.DYN_INPUT_FILES_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.ENV_REPLACE_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.ENV_REPLACE_NO_RECUR_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.EXE_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.IMPORT_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEYREF_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEY_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEY_GEN_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.METADATA_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.NAMESPACE_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.OUTPUT_TAG;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.PATH_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.SHELL_TYPE_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.SPLIT_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.VAL_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.WORKFLOW_MET_ATTR;
-import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.WRITER_CLASS_ATTR;
 import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.CONFIG_FILE_PATH;
-import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.QUERY_FILE_MANAGER_URL;
+import static org.apache.oodt.cas.pge.util.XmlHelper.fillIn;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getCustomMetadataElement;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getDir;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getDynamicConfigFiles;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getExeCmds;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getFileStaging;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getImports;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getMetadataElements;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getMetadataKey;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getMetadataKeyRef;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getMetadataValues;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getOuputDirs;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getOutput;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getRootElement;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getShellType;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getStageFilesMetKeys;
+import static org.apache.oodt.cas.pge.util.XmlHelper.isForceStage;
+import static org.apache.oodt.cas.pge.util.XmlHelper.isWorkflowMetKey;
 
 //JDK imports
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.LinkedList;
 import java.util.List;
-import java.util.Vector;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 //OODT imports
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
-import org.apache.oodt.cas.filemgr.util.QueryUtils;
-import org.apache.oodt.cas.filemgr.util.SqlParser;
 import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.metadata.util.PathUtils;
 import org.apache.oodt.cas.pge.metadata.PgeMetadata;
-import org.apache.oodt.commons.xml.XMLUtils;
+import org.apache.oodt.cas.pge.util.Pair;
+import org.apache.oodt.cas.pge.util.XmlHelper;
 
-//W3C imports
-import org.w3c.dom.DOMException;
+//DOM imports
 import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
+
+import com.google.common.collect.Lists;
 
 /**
  * An implementation of the {@link PgeConfigBuilder} that reads an XML file
@@ -80,239 +61,167 @@ import org.w3c.dom.NodeList;
  * @author bfoster (Brian Foster)
  */
 public class XmlFilePgeConfigBuilder implements PgeConfigBuilder {
-    
-    private static Logger LOG = Logger.getLogger(XmlFilePgeConfigBuilder.class.getName());
-    
-    public PgeConfig build(PgeMetadata pgeMetadata) throws IOException {
-        try {
-            PgeConfig pgeConfig = new PgeConfig();
-            buildImports(
-               fillIn(pgeMetadata.getMetadata(CONFIG_FILE_PATH),
-                     pgeMetadata.asMetadata()), null, pgeConfig, pgeMetadata);
-            return pgeConfig;
-        } catch (Exception e) {
-            throw new IOException("Failed to build PgeConfig : "
-                    + e.getMessage(), e);
-        }
-    }
-    
-    private void buildImports(String configFile, String namespace, PgeConfig pgeConfig, PgeMetadata pgeMetadata) throws MalformedURLException, Exception {
-        
-        Element root = this.getRootElement(configFile);
-        
-         // load parent files
-        NodeList nodeList = root.getElementsByTagName(IMPORT_TAG);
-        for (int i = 0; i < nodeList.getLength(); i++) {
-            String curImportNS = this.fillIn(((Element) nodeList.item(i))
-                    .getAttribute(NAMESPACE_ATTR), pgeMetadata
-                    .asMetadata());
-            String file = this.fillIn(((Element) nodeList.item(i))
-                    .getAttribute(FILE_ATTR), pgeMetadata.asMetadata());
-            if (!file.startsWith(File.separator))
-                file = new File(configFile).getParent()
-                        + File.separator + file;
-            this.buildImports(file, curImportNS.equals("") ? null : curImportNS, pgeConfig, pgeMetadata);
-        }
-
-        // load base config file
-        LOG.log(Level.INFO, "Loading PgeConfig file '" + configFile + "'");
-        this.build(root, namespace, pgeConfig, pgeMetadata);
-    }
-
-    private void build(Element root, String namespace, PgeConfig pgeConfig,
-            PgeMetadata pgeMetadata) throws MalformedURLException, Exception {
-
-        // load custom metadata
-        PgeMetadata localPgeMetadata = this.getCustomMetadata((Element) root
-                .getElementsByTagName(CUSTOM_METADATA_TAG).item(0), pgeMetadata);
-        PgeMetadata curPgeMetadata = new PgeMetadata();
-        curPgeMetadata.replaceMetadata(pgeMetadata);
-        curPgeMetadata.replaceMetadata(localPgeMetadata);
-        Metadata curMetadata = curPgeMetadata.asMetadata();
-        
-        // load dynamic config file info
-        List<DynamicConfigFile> configFileList = this.getDynConfigFile(
-                (Element) root.getElementsByTagName(DYN_INPUT_FILES_TAG)
-                        .item(0), curMetadata);
-        for (DynamicConfigFile dcf : configFileList)
-            pgeConfig.addDynamicConfigFile(dcf);
-
-        // load exe info
-        if (root.getElementsByTagName(EXE_TAG).getLength() > 0) {
-            pgeConfig.setExeDir(this.getExeDir((Element) root.getElementsByTagName(
-                EXE_TAG).item(0), curMetadata));        
-            pgeConfig.setShellType(this.getShellType((Element) root
-                .getElementsByTagName(EXE_TAG).item(0), curMetadata));
-            List<String> exeCmds = pgeConfig.getExeCmds();
-            exeCmds.addAll(this.getExeCmds((Element) root.getElementsByTagName(
-                    EXE_TAG).item(0), curMetadata));
-            pgeConfig.setExeCmds(exeCmds);
-        }
-
-        // load output dirs
-        List<OutputDir> outputDirs = this.getOuputDirs((Element) root
-                .getElementsByTagName(OUTPUT_TAG).item(0), curMetadata);
-        for (OutputDir outputDir : outputDirs)
+
+   private final List<String> missingMetadataKeys;
+
+   public XmlFilePgeConfigBuilder() {
+      missingMetadataKeys = Lists.newArrayList();
+   }
+
+   @Override
+   public PgeConfig build(PgeMetadata pgeMetadata) throws IOException {
+      try {
+         missingMetadataKeys.clear();
+
+         PgeConfig pgeConfig = new PgeConfig();
+         String configFile = fillIn(pgeMetadata.getMetadata(CONFIG_FILE_PATH),
+               pgeMetadata.asMetadata());
+         if (configFile == null) {
+            throw new Exception("Must specify metadata field '"
+                  + CONFIG_FILE_PATH + "'");
+         }
+         pgeMetadata.replaceMetadata(loadConfigFile(configFile, pgeConfig,
+               pgeMetadata));
+         return pgeConfig;
+      } catch (Exception e) {
+         throw new IOException("Failed to build PgeConfig : " + e.getMessage(),
+               e);
+      }
+   }
+
+   private PgeMetadata loadConfigFile(String configFile, PgeConfig pgeConfig,
+         PgeMetadata parentPgeMetadata) throws Exception {
+      PgeMetadata pgeMetadata = new PgeMetadata(parentPgeMetadata);
+      Element root = getRootElement(configFile);
+
+      // Read in imports
+      List<Pair<String, String>> imports = getImports(root, pgeMetadata.asMetadata());
+      for (Pair<String, String> imp : imports) {
+         String namespace = imp.getFirst();
+         String file = imp.getSecond();
+
+         // If relative path, then make path relative to configFile.
+         if (!file.startsWith(File.separator)) {
+            file = new File(configFile).getParent() + File.separator + file;
+         }
+
+         // Add metadata generated from import.
+         if (namespace != null) {
+            pgeMetadata.replaceMetadata(
+                  loadConfigFile(file, pgeConfig, parentPgeMetadata),
+                  namespace);
+         } else {
+            pgeMetadata.replaceMetadata(loadConfigFile(file, pgeConfig,
+                  parentPgeMetadata));
+         }
+      }
+
+      // load custom metadata
+      loadCustomMetadata(root, pgeMetadata);
+      Metadata metadata = pgeMetadata.asMetadata();
+
+      // load dynamic config file info
+      for (DynamicConfigFile dcf : getDynamicConfigFiles(root, metadata)) {
+         pgeConfig.addDynamicConfigFile(dcf);
+      }
+
+      // load file staging info.
+      loadFileStagingInfo(root, pgeConfig, pgeMetadata);
+
+      // load exe info
+      Element exeElem = XmlHelper.getExe(root);
+      if (exeElem != null) {
+         pgeConfig.setExeDir(getDir(exeElem, metadata));
+         pgeConfig.setShellType(getShellType(exeElem, metadata));
+         pgeConfig.setExeCmds(getExeCmds(exeElem, metadata));
+      }
+
+      // load output dirs
+      Element outputElem = getOutput(root);
+      if (outputElem != null) {
+         for (OutputDir outputDir : getOuputDirs(outputElem, metadata)) {
             pgeConfig.addOuputDirAndExpressions(outputDir);
-        
-        // add local pge metadata to global pge metadata with given namespace
-        if (namespace != null) {
-           pgeMetadata.replaceMetadata(localPgeMetadata, namespace);
-        } else {
-           pgeMetadata.replaceMetadata(localPgeMetadata);  
-        }
-    }
-
-    private PgeMetadata getCustomMetadata(Element customMetadataElem, PgeMetadata pgeMetadata)
-            throws MalformedURLException, Exception {
-    	PgeMetadata localPgeMetadata = new PgeMetadata();
-    	PgeMetadata curPgeMetadata = new PgeMetadata();
-    	curPgeMetadata.replaceMetadata(pgeMetadata);
-    	Metadata curPlusLocalMetadata = new Metadata();
-    	curPlusLocalMetadata.addMetadata(curPgeMetadata.asMetadata().getHashtable());
-    	
-        if (customMetadataElem != null) {
-            NodeList customMetadataList = customMetadataElem
-                    .getElementsByTagName(METADATA_TAG);
-            for (int i = 0; i < customMetadataList.getLength(); i++) {
-                Element metadataElement = (Element) customMetadataList.item(i);
-                String key = metadataElement.getAttribute(KEY_ATTR);
-                if (key.equals(""))
-                	key = this.fillIn(metadataElement.getAttribute(KEY_GEN_ATTR), curPlusLocalMetadata);
-                if (!metadataElement.getAttribute(KEYREF_ATTR).equals("")) {
-                	String val = metadataElement.getAttribute(KEYREF_ATTR);
-                    if (metadataElement.getAttribute(ENV_REPLACE_NO_RECUR_ATTR)
-                            .toLowerCase().equals("true"))
-                        val = this.fillIn(val, curPlusLocalMetadata, false);
-                    else if (!metadataElement.getAttribute(ENV_REPLACE_ATTR)
-                            .toLowerCase().equals("false"))
-                        val = this.fillIn(val, curPlusLocalMetadata);
-                	localPgeMetadata.linkKey(key, val);
-                	curPgeMetadata.linkKey(key, val);
-                }else {
-                	String val = metadataElement.getAttribute(VAL_ATTR);
-                	if (val.equals("")) 
-                		val = metadataElement.getTextContent();
-                    if (metadataElement.getAttribute(ENV_REPLACE_NO_RECUR_ATTR)
-                            .toLowerCase().equals("true"))
-                        val = this.fillIn(val, curPlusLocalMetadata, false);
-                    else if (!metadataElement.getAttribute(ENV_REPLACE_ATTR)
-                            .toLowerCase().equals("false"))
-                        val = this.fillIn(val, curPlusLocalMetadata);
-                    List<String> valList = new Vector<String>();
-                    if (!metadataElement.getAttribute(SPLIT_ATTR)
-                            .toLowerCase().equals("false")) 
-                    	valList.addAll(Arrays.asList((val + ",").split(",")));
-                    else 
-                    	valList.add(val);
-                    localPgeMetadata.replaceMetadata(key, valList);
-                    curPgeMetadata.replaceMetadata(key, valList);
-                }
-                if (metadataElement.getAttribute(WORKFLOW_MET_ATTR)
-                        .toLowerCase().equals("true"))
-                	localPgeMetadata.markAsDynamicMetadataKey(key);
-                
-                List<String> values = curPgeMetadata.getAllMetadata(key);
-                if (values != null) {
-                   curPlusLocalMetadata.replaceMetadata(key, values);
-                }
-            }
-        }
-        return localPgeMetadata;
-    }
-
-    private Element getRootElement(String xmlFilePath)
-            throws FileNotFoundException {
-        return XMLUtils.getDocumentRoot(
-                new FileInputStream(new File(xmlFilePath)))
-                .getDocumentElement();
-    }
-
-    private List<DynamicConfigFile> getDynConfigFile(
-            Element dynConfigFileElement, Metadata curMetadata)
-            throws MalformedURLException, Exception {
-        List<DynamicConfigFile> configFileList = new LinkedList<DynamicConfigFile>();
-        if (dynConfigFileElement != null) {
-            NodeList dynConfigFilesList = dynConfigFileElement
-                    .getElementsByTagName(FILE_TAG);
-            for (int i = 0; i < dynConfigFilesList.getLength(); i++) {
-                Element fileElement = (Element) dynConfigFilesList.item(i);
-                configFileList.add(new DynamicConfigFile(this.fillIn(
-                        fileElement.getAttribute(PATH_ATTR), curMetadata), this
-                        .fillIn(fileElement.getAttribute(WRITER_CLASS_ATTR),
-                                curMetadata), (Object[]) this.fillIn(
-                        fileElement.getAttribute(ARGS_ATTR), curMetadata)
-                        .split(",")));
-            }
-        }
-        return configFileList;
-    }
-
-    private String getExeDir(Element exeElement, Metadata curMetadata)
-            throws MalformedURLException, Exception {
-        if (exeElement != null)
-            return this.fillIn(exeElement.getAttribute(DIR_ATTR), curMetadata);
-        else
-            return null;
-    }
-
-    private String getShellType(Element exeElement, Metadata curMetadata)
-            throws MalformedURLException, Exception {
-        if (exeElement != null)
-            return this.fillIn(exeElement.getAttribute(SHELL_TYPE_ATTR),
-                    curMetadata);
-        else
-            return null;
-    }
-
-    private List<String> getExeCmds(Element exeElement, Metadata curMetadata)
-            throws MalformedURLException, DOMException, Exception {
-        LinkedList<String> exeCmds = new LinkedList<String>();
-        if (exeElement != null) {
-            NodeList cmds = exeElement.getElementsByTagName(CMD_TAG);
-            for (int i = 0; i < cmds.getLength(); i++) {
-                Element cmd = (Element) cmds.item(i);
-                exeCmds.add(this.fillIn(cmd.getTextContent(), curMetadata));
-            }
-        }
-        return exeCmds;
-    }
-
-    private List<OutputDir> getOuputDirs(Element ouputDirElement,
-            Metadata curMetadata) throws MalformedURLException, Exception {
-        List<OutputDir> outputDirs = new LinkedList<OutputDir>();
-        if (ouputDirElement != null) {
-            NodeList outputDirsList = ouputDirElement
-                    .getElementsByTagName(DIR_TAG);
-            for (int i = 0; i < outputDirsList.getLength(); i++) {
-                Element outputDirElement = (Element) outputDirsList.item(i);
-                String dirPath = fillIn(outputDirElement
-                      .getAttribute(PATH_ATTR), curMetadata);
-                OutputDir outputDir = new OutputDir(dirPath,
-                      Boolean.parseBoolean(fillIn(outputDirElement
-                            .getAttribute(CREATE_BEFORE_EXEC_ATTR),
-                            curMetadata)));
-                outputDirs.add(outputDir);
+         }
+      }
+
+      return pgeMetadata;
+   }
+
+   private void loadCustomMetadata(Element root, PgeMetadata pgeMetadata)
+         throws MalformedURLException, Exception {
+
+      // Check if there is a 'customMetadata' elem and load it.
+      Element customMetadataElem = getCustomMetadataElement(root);
+      if (customMetadataElem == null) {
+         return;
+      }
+
+      // Iterate through metadata elements.
+      for (Element metElem : getMetadataElements(customMetadataElem)) {
+         Metadata curMetadata = pgeMetadata.asMetadata();
+
+         // Load supported metadata element attributes.
+         String key = getMetadataKey(metElem, curMetadata);
+         List<String> values = getMetadataValues(metElem, curMetadata);
+         String keyRef = getMetadataKeyRef(metElem, curMetadata);
+
+         // Check that either val or key-ref is given.
+         if (!values.isEmpty() && keyRef != null) {
+            throw new Exception(
+                  "Cannot specify both values and keyref for metadata key '"
+                        + key + "'");
+
+            // If val is given then set metadata with key and val.
+         } else if (!values.isEmpty()) {
+            pgeMetadata.replaceMetadata(key, values);
+
+            // Otherwise key-ref was given, so set the link.
+         } else {
+            pgeMetadata.linkKey(key, keyRef);
+         }
+
+         // Check if current key should be marked as workflow metadata.
+         if (isWorkflowMetKey(metElem, curMetadata)) {
+            pgeMetadata.markAsDynamicMetadataKey(key);
+         }
+      }
+   }
+
+   private void loadFileStagingInfo(Element root, PgeConfig pgeConfig,
+         PgeMetadata pgeMetadata) throws Exception {
+      Metadata metadata = pgeMetadata.asMetadata();
+      Element fileStagingElem = getFileStaging(root);
+
+      // Check if there is file staging info specified.
+      if (fileStagingElem != null) {
+         FileStagingInfo fileStagingInfo = new FileStagingInfo(getDir(
+               fileStagingElem, metadata), isForceStage(fileStagingElem,
+               metadata));
+
+         // Iterate through list of metadata keys which have list of files as
+         // their values which should be staged.
+         for (String metKey : getStageFilesMetKeys(fileStagingElem, metadata)) {
+            List<String> files = metadata.getAllMetadata(metKey);
+            fileStagingInfo.addFilePaths(files);
+
+            // Generate paths which the files will be staged to.
+            List<String> newPaths = Lists.newArrayList();
+            for (String file : files) {
+               File fileHandle = new File(file);
+               if (fileStagingInfo.isForceStaging() || !fileHandle.exists()) {
+                  newPaths.add(fileStagingInfo.getStagingDir() + File.separator
+                        + fileHandle.getName());
+               } else {
+                  newPaths.add(file);
+               }
             }
-        }
-        return outputDirs;
-    }
-
-    private String fillIn(String value, Metadata inputMetadata) throws Exception {
-    	return this.fillIn(value, inputMetadata, true);
-    }
-    
-    private String fillIn(String value, Metadata inputMetadata, boolean envReplaceRecur)
-            throws Exception {
-    	try {
-	        while ((value = PathUtils.doDynamicReplacement(value, inputMetadata)).contains("[") && envReplaceRecur);
-	        if (value.toUpperCase()
-	                .matches("^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$"))
-	            value = QueryUtils.getQueryResultsAsString(new XmlRpcFileManagerClient(new URL(inputMetadata
-	                    .getMetadata(QUERY_FILE_MANAGER_URL.getName()))).complexQuery(SqlParser.parseSqlQueryMethod(value)));
-	        return value;
-    	}catch (Exception e) {
-    		throw new Exception("Failed to parse value: " + value, e);
-    	}
-    }
+
+            // Update metadata key with what will be the new paths of the files.
+            pgeMetadata.replaceMetadata(metKey, newPaths);
+         }
+
+         // Add staging info to PgeConfig.
+         pgeConfig.setFileStagingInfo(fileStagingInfo);
+      }
+   }
 }

Modified: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java?rev=1351905&r1=1351904&r2=1351905&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetKeys.java Wed Jun 20 00:23:59 2012
@@ -21,7 +21,7 @@ import com.google.common.annotations.Vis
 
 /**
  * PGETaskInstance Reserved Metadata keys.
- * 
+ *
  * @author bfoster (Brian Foster)
  */
 public enum PgeTaskMetKeys {
@@ -57,14 +57,12 @@ public enum PgeTaskMetKeys {
          "PGETask/PropertyAdders",
          "PGETask_PropertyAdderClasspath",
          true),
-
    /**
     * {@link FileStager}s classpath.
     */
    FILE_STAGER(
          "PGETask/FileStager",
          "PGETask/FileStager"),
-
    /**
     * List of {@link ConfigFilePropertyAdder}s classpaths to be run.
     */

Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/Pair.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/Pair.java?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/Pair.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/Pair.java Wed Jun 20 00:23:59 2012
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.util;
+
+/**
+ * Wrapper class around 2 objects.
+ *
+ * @author bfoster (Brian Foster)
+ */
+public class Pair<Type1, Type2> {
+
+   private final Type1 first;
+   private final Type2 second;
+
+   public Pair(Type1 first, Type2 second) {
+      this.first = first;
+      this.second = second;
+   }
+
+   public Type1 getFirst() {
+      return first;
+   }
+
+   public Type2 getSecond() {
+      return second;
+   }
+}

Propchange: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/Pair.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java Wed Jun 20 00:23:59 2012
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.util;
+
+//OODT static imports
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.ARGS_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.CMD_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.CREATE_BEFORE_EXEC_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.CUSTOM_METADATA_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.DIR_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.DIR_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.DYN_INPUT_FILES_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.ENV_REPLACE_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.ENV_REPLACE_NO_RECUR_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.EXE_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_STAGING_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FILE_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.FORCE_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.IMPORT_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEYREF_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEY_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.KEY_GEN_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.METADATA_KEY_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.METADATA_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.NAMESPACE_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.OUTPUT_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.PATH_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.SHELL_TYPE_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.SPLIT_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.STAGE_FILES_TAG;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.VAL_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.WORKFLOW_MET_ATTR;
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.WRITER_CLASS_ATTR;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.QUERY_FILE_MANAGER_URL;
+
+//JDK imports
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.util.QueryUtils;
+import org.apache.oodt.cas.filemgr.util.SqlParser;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.oodt.cas.pge.config.DynamicConfigFile;
+import org.apache.oodt.cas.pge.config.OutputDir;
+import org.apache.oodt.commons.xml.XMLUtils;
+
+//DOM imports
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+//Google imports
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+/**
+ * Help class with method for parsing XMLFilePgeConfigBuilder config XML file.
+ *
+ * @author bfoster (Brian Foster)
+ */
+public class XmlHelper {
+
+   private XmlHelper() {}
+
+   public static Element getRootElement(String xmlFilePath)
+         throws FileNotFoundException {
+      return XMLUtils.getDocumentRoot(
+            new FileInputStream(new File(xmlFilePath))).getDocumentElement();
+   }
+
+   public static List<Pair<String, String>> getImports(Element elem, Metadata metadata) throws Exception {
+      List<Pair<String, String>> imports = Lists.newArrayList();
+      NodeList nodeList = elem.getElementsByTagName(IMPORT_TAG);
+      for (int i = 0; i < nodeList.getLength(); i++) {
+         Element importElem = (Element) nodeList.item(i);
+         String namespace = getNamespace(importElem, metadata);
+         String file = getFile(importElem, metadata);
+         imports.add(new Pair<String, String>(namespace, file));
+      }
+      return imports;
+   }
+
+   public static String getNamespace(Element elem, Metadata metadata) throws Exception {
+      String namespace = elem.getAttribute(NAMESPACE_ATTR);
+      if (!Strings.isNullOrEmpty(namespace)) {
+         return fillIn(namespace, metadata, false);
+      } else {
+         return null;
+      }
+   }
+
+   public static String getFile(Element elem, Metadata metadata) throws Exception {
+      String file = elem.getAttribute(FILE_ATTR);
+      if (!Strings.isNullOrEmpty(file)) {
+         return fillIn(file, metadata, false);
+      } else {
+         return null;
+      }
+   }
+
+   public static Element getCustomMetadataElement(Element root) throws Exception {
+      NodeList nodes = root.getElementsByTagName(CUSTOM_METADATA_TAG);
+      if (nodes.getLength() == 0) {
+         return null;
+      } else if (nodes.getLength() == 1) {
+         return (Element) nodes.item(0);
+      } else {
+         throw new Exception("Found more than one '" + CUSTOM_METADATA_TAG
+               + "' element");
+      }
+   }
+
+   public static List<Element> getMetadataElements(Element customMetadataElem) {
+      NodeList metElemsNodeList = customMetadataElem
+         .getElementsByTagName(METADATA_TAG);
+      List<Element> metElems = Lists.newArrayList();
+      for (int i = 0; i < metElemsNodeList.getLength(); i++) {
+         metElems.add((Element) metElemsNodeList.item(i));
+      }
+      return metElems;
+   }
+
+   public static String getMetadataKey(Element metElem, Metadata metadata) throws Exception {
+      String key = metElem.getAttribute(KEY_ATTR);
+
+      // no key attr, so check for key_gen attr.
+      if (Strings.isNullOrEmpty(key)) {
+         key = fillIn(metElem.getAttribute(KEY_GEN_ATTR), metadata);
+      }
+
+      // if still no key value, then fail.
+      if (Strings.isNullOrEmpty(key)) {
+         throw new Exception("Must specify either metadata attr '" + KEY_ATTR
+               + "' or '" + KEY_GEN_ATTR + "'");
+      }
+
+      // else success!
+      return key;
+   }
+
+   public static boolean isEnvReplaceNoRecur(Element elem, Metadata metadata)
+         throws Exception {
+      String isEnvReplaceNoRecur = elem.getAttribute(ENV_REPLACE_NO_RECUR_ATTR);
+      if (Strings.isNullOrEmpty(isEnvReplaceNoRecur)) {
+         return false;
+      } else {
+         return isEnvReplaceNoRecur.trim().toLowerCase().equals("true");
+      }
+   }
+
+   public static boolean isEnvReplace(Element elem, Metadata metadata)
+         throws Exception {
+      String isEnvReplace = elem.getAttribute(ENV_REPLACE_ATTR);
+      if (Strings.isNullOrEmpty(isEnvReplace)) {
+         return true;
+      } else {
+         return !isEnvReplace.trim().toLowerCase().equals("false");
+      }
+   }
+
+   public static boolean isMultiValue(Element elem, Metadata metadata)
+         throws Exception {
+      return Boolean.parseBoolean(fillIn(elem.getAttribute(SPLIT_ATTR),
+            metadata));
+   }
+
+   public static List<String> getMetadataValues(Element elem, Metadata metadata)
+         throws Exception {
+      List<String> values = Lists.newArrayList();
+
+      // Read val attr.
+      String value = elem.getAttribute(VAL_ATTR);
+
+      // Check if val tag was not specified see if value was given as element
+      // text.
+      if (Strings.isNullOrEmpty(value)) {
+        value = elem.getTextContent();
+      }
+
+      // If value was found.
+      if (!Strings.isNullOrEmpty(value)) {
+
+         // Is multi-value so split up value.
+         if (isMultiValue(elem, metadata)) {
+            for (String v : Splitter.on(",").split(value)) {
+
+               // Check for envReplace and perform met replacement on value if set.
+               if (isEnvReplaceNoRecur(elem, metadata)) {
+                  values.add(fillIn(v, metadata, false));
+               } else if (isEnvReplace(elem, metadata)) {
+                  values.add(fillIn(v, metadata));
+               }
+            }
+
+         // Is scalar
+         } else {
+
+            // Check for envReplace and perform met replacement on value if set.
+            if (isEnvReplaceNoRecur(elem, metadata)) {
+               value = fillIn(value, metadata, false);
+            } else if (isEnvReplace(elem, metadata)) {
+               value = fillIn(value, metadata);
+            }
+            values.add(value);
+         }
+
+      }
+      return values;
+   }
+
+   public static String getMetadataKeyRef(Element elem, Metadata metadata)
+         throws Exception {
+      String keyRef = elem.getAttribute(KEYREF_ATTR);
+      if (!Strings.isNullOrEmpty(keyRef)) {
+
+         // Check for envReplace and perform met replacement on value if set.
+         if (isEnvReplaceNoRecur(elem, metadata)) {
+            keyRef = fillIn(keyRef, metadata, false);
+         } else if (isEnvReplace(elem, metadata)) {
+            keyRef = fillIn(keyRef, metadata);
+         }
+
+         return keyRef;
+      } else {
+         return null;
+      }
+   }
+
+   public static boolean isWorkflowMetKey(Element elem, Metadata metadata) throws Exception {
+      return Boolean.parseBoolean(fillIn(elem.getAttribute(WORKFLOW_MET_ATTR), metadata, false));
+   }
+
+   public static String getPath(Element elem, Metadata metadata) throws Exception {
+      return fillIn(elem.getAttribute(PATH_ATTR), metadata, false);
+   }
+
+   public static String getWriter(Element elem, Metadata metadata) throws Exception {
+      return fillIn(elem.getAttribute(WRITER_CLASS_ATTR), metadata, false);
+   }
+
+   public static List<String> getArgs(Element elem, Metadata metadata)
+         throws Exception {
+      List<String> args = Lists.newArrayList();
+      for (String arg : Splitter.on(",").split(elem.getAttribute(ARGS_ATTR))) {
+         args.add(fillIn(arg, metadata, false));
+      }
+      if (args.size() == 1 && Strings.isNullOrEmpty(args.get(0))) {
+         return Lists.newArrayList();
+      } else {
+         return args;
+      }
+   }
+
+   public static List<DynamicConfigFile> getDynamicConfigFiles(Element elem,
+         Metadata metadata) throws Exception {
+      List<DynamicConfigFile> dynamicConfigFiles = Lists.newArrayList();
+      NodeList nodeList = elem.getElementsByTagName(DYN_INPUT_FILES_TAG);
+
+      // Check if dynInput element exists.
+      if (nodeList.getLength() > 0) {
+         Element dynamicConfigFilesElem = (Element) nodeList.item(0);
+         nodeList = dynamicConfigFilesElem.getElementsByTagName(FILE_TAG);
+
+         // Load each dynamic input file information.
+         for (int i = 0; i < nodeList.getLength(); i++) {
+            Element fileElem = (Element) nodeList.item(i);
+            String path = getPath(fileElem, metadata);
+            String writer = getWriter(fileElem, metadata);
+            List<String> args = getArgs(fileElem, metadata);
+            dynamicConfigFiles.add(new DynamicConfigFile(path, writer, args
+                  .toArray(new String[0])));
+         }
+      }
+      return dynamicConfigFiles;
+   }
+
+   public static Element getExe(Element elem) throws Exception {
+      NodeList nodeList = elem.getElementsByTagName(EXE_TAG);
+      if (nodeList.getLength() > 1) {
+         throw new Exception("Can only specify '" + EXE_TAG + "' once!");
+      } else if (nodeList.getLength() == 1) {
+         return (Element) nodeList.item(0);
+      } else {
+         return null;
+      }
+   }
+
+   public static String getDir(Element elem, Metadata metadata)
+         throws MalformedURLException, Exception {
+      return fillIn(elem.getAttribute(DIR_ATTR), metadata);
+   }
+
+   public static String getShellType(Element elem, Metadata metadata)
+         throws MalformedURLException, Exception {
+      return fillIn(elem.getAttribute(SHELL_TYPE_ATTR), metadata);
+   }
+
+   public static List<String> getExeCmds(Element elem, Metadata metadata)
+         throws MalformedURLException, DOMException, Exception {
+      List<String> exeCmds = Lists.newArrayList();
+      NodeList nodeList = elem.getElementsByTagName(CMD_TAG);
+      for (int i = 0; i < nodeList.getLength(); i++) {
+         Element cmdElem = (Element) nodeList.item(i);
+         String exeCmd = cmdElem.getTextContent();
+         if (isEnvReplaceNoRecur(cmdElem, metadata)) {
+            exeCmd = fillIn(exeCmd, metadata, false);
+         } else if (isEnvReplace(cmdElem, metadata)) {
+            exeCmd = fillIn(exeCmd, metadata);
+         }
+         exeCmds.add(exeCmd);
+      }
+      return exeCmds;
+   }
+
+   public static Element getFileStaging(Element elem) throws Exception {
+      NodeList nodeList = elem.getElementsByTagName(FILE_STAGING_TAG);
+      if (nodeList.getLength() > 1) {
+         throw new Exception("Can only specify '" + FILE_STAGING_TAG + "' once!");
+      } else if (nodeList.getLength() == 1) {
+         return (Element) nodeList.item(0);
+      } else {
+         return null;
+      }
+   }
+
+   public static boolean isForceStage(Element elem, Metadata metadata) throws Exception {
+      return Boolean.parseBoolean(fillIn(elem.getAttribute(FORCE_ATTR), metadata));
+   }
+
+   public static String getFileStagingMetadataKey(Element elem, Metadata metadata) throws Exception {
+      return fillIn(elem.getAttribute(METADATA_KEY_ATTR), metadata);
+   }
+
+   public static List<String> getStageFilesMetKeys(Element elem, Metadata metadata) throws Exception {
+      List<String> metKeys = Lists.newArrayList();
+      NodeList nodeList = elem.getElementsByTagName(STAGE_FILES_TAG);
+      for (int i = 0; i < nodeList.getLength(); i++) {
+         Element stageFilesElem = (Element) nodeList.item(i);
+         metKeys.add(getFileStagingMetadataKey(stageFilesElem, metadata));
+      }
+      return metKeys;
+   }
+
+   public static Element getOutput(Element elem) throws Exception {
+      NodeList nodeList = elem.getElementsByTagName(OUTPUT_TAG);
+      if (nodeList.getLength() > 1) {
+         throw new Exception("Can only specify '" + OUTPUT_TAG + "' once!");
+      } else if (nodeList.getLength() == 1) {
+         return (Element) nodeList.item(0);
+      } else {
+         return null;
+      }
+   }
+
+   public static boolean isCreateBeforeExe(Element elem, Metadata metadata) throws Exception {
+      return Boolean.parseBoolean(fillIn(elem.getAttribute(CREATE_BEFORE_EXEC_ATTR), metadata));
+   }
+
+   public static List<OutputDir> getOuputDirs(Element elem, Metadata metadata) throws Exception {
+      List<OutputDir> outputDirs = Lists.newArrayList();
+      NodeList nodeList = elem.getElementsByTagName(DIR_TAG);
+      for (int i = 0; i < nodeList.getLength(); i++) {
+         Element outputDirElem = (Element) nodeList.item(i);
+         String path = getPath(outputDirElem, metadata);
+         boolean createBeforeExe = isCreateBeforeExe(outputDirElem, metadata);
+         outputDirs.add(new OutputDir(path, createBeforeExe));
+      }
+      return outputDirs;
+   }
+
+   public static String fillIn(String value, Metadata inputMetadata) throws Exception {
+      return fillIn(value, inputMetadata, true);
+   }
+
+   public static String fillIn(String value, Metadata inputMetadata,
+         boolean envReplaceRecur) throws Exception {
+      try {
+         while ((value = PathUtils.doDynamicReplacement(value, inputMetadata))
+               .contains("[") && envReplaceRecur)
+            ;
+         if (value.toUpperCase().matches(
+               "^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$"))
+            value = QueryUtils
+                  .getQueryResultsAsString(new XmlRpcFileManagerClient(new URL(
+                        inputMetadata.getMetadata(QUERY_FILE_MANAGER_URL
+                              .getName()))).complexQuery(SqlParser
+                        .parseSqlQueryMethod(value)));
+         return value;
+      } catch (Exception e) {
+         throw new Exception("Failed to parse value: " + value, e);
+      }
+   }
+}

Propchange: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java Wed Jun 20 00:23:59 2012
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.writers;
+
+//Google static imports
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+//JDK imports
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+//Google imports
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * CSV {@link SciPgeConfigFileWriter} which takes a comma separted list of
+ * {@link String}s as the first index of the args and generates a CSV file from
+ * the metadata fields listed. For example, if the list of {@link String}s given
+ * is:
+ *
+ * <pre>
+ *    InputFiles,IsText
+ * </pre>
+ *
+ * And:
+ *
+ * <pre>
+ *    InputFiles=File1.txt,File2.dat,File3.xml
+ *    IsText=true,false,true
+ * </pre>
+ *
+ * Then the CSV file create will look like:
+ *
+ * <pre>
+ *    InputFiles,IsText
+ *    File1.text,true
+ *    File2.dat,false
+ *    File3.xml,true
+ * </pre>
+ *
+ * @author bfoster (Brian Foster)
+ */
+public class CsvConfigFileWriter implements SciPgeConfigFileWriter {
+
+   private static final int HEADER_INDEX = 0;
+   private static final int DELIM_INDEX = 0;
+
+   private static final String DEFAULT_DELIM = ",";
+
+   @Override
+   public File createConfigFile(String sciPgeConfigFilePath,
+         Metadata inputMetadata, Object... customArgs) throws IOException {
+      checkArgument(customArgs.length > 0,
+            CsvConfigFileWriter.class.getCanonicalName()
+                  + " has no args specified");
+      List<String> header = checkNotNull(
+            Lists.newArrayList(Splitter.on(",").split(
+                  (String) customArgs[HEADER_INDEX])),
+            "Must specify CSV header in args at index = '" + HEADER_INDEX + "'");
+      String delim = DEFAULT_DELIM;
+      if (customArgs.length > DELIM_INDEX) {
+         delim = (String) customArgs[DELIM_INDEX];
+      }
+
+      return writeCsvFile(sciPgeConfigFilePath, header,
+            generateRows(header, inputMetadata), delim);
+   }
+
+   @VisibleForTesting
+   protected List<List<String>> generateRows(List<String> header,
+         Metadata meatadata) {
+      List<List<String>> rows = Lists.newArrayList();
+      int index = 0;
+      TOP: while (true) {
+         for (String columnName : header) {
+            List<String> values = meatadata.getAllMetadata(columnName);
+            if (values.size() <= index) {
+               break TOP;
+            }
+            List<String> row = rows.get(index);
+            if (row == null) {
+               row = Lists.newArrayList();
+            }
+            row.add(values.get(index));
+            rows.set(index, row);
+         }
+      }
+      return rows;
+   }
+
+   @VisibleForTesting
+   protected File writeCsvFile(String file, List<String> header,
+         List<List<String>> rows, String delim) throws IOException {
+      File outputFile = new File(file);
+      PrintStream ps = null;
+      try {
+         ps = new PrintStream(new FileOutputStream(outputFile));
+         ps.println(Joiner.on(delim).join(header));
+         for (List<String> row : rows) {
+            ps.println(Joiner.on(delim).join(row));
+         }
+         return outputFile;
+      } catch (Exception e) {
+         throw new IOException("Failed to write CSV file '" + file + "' : "
+               + e.getMessage(), e);
+      } finally {
+         try {
+            ps.close();
+         } catch (Exception ignore) {
+         }
+      }
+   }
+}

Propchange: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/CsvConfigFileWriter.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java (added)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java Wed Jun 20 00:23:59 2012
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.writers;
+
+//Google static imports
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+//OODT static imports
+import static org.apache.oodt.cas.pge.util.XmlHelper.fillIn;
+
+//JDK imports
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+//Google imports
+import com.google.common.annotations.VisibleForTesting;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * Text {@link SciPgeConfigFileWriter} which takes a template with envReplace
+ * metadata that gets dynamically filled out and written to a file.
+ *
+ * @author bfoster (Brian Foster)
+ */
+public class TextConfigFileWriter implements SciPgeConfigFileWriter {
+
+   private static final int TEMPLATE_INDEX = 0;
+
+   @Override
+   public File createConfigFile(String sciPgeConfigFilePath,
+         Metadata inputMetadata, Object... customArgs) throws IOException {
+      checkArgument(customArgs.length > 0,
+            TextConfigFileWriter.class.getCanonicalName()
+                  + " has no args specified");
+      String template = checkNotNull((String) customArgs[TEMPLATE_INDEX],
+            "Must specify Text file template in args at index = '"
+                  + TEMPLATE_INDEX + "'");
+
+      try {
+         return writeTextFile(sciPgeConfigFilePath, fillIn(template, inputMetadata));
+      } catch (Exception e) {
+         throw new IOException(e);
+      }
+   }
+
+   @VisibleForTesting
+   protected File writeTextFile(String file, String text) throws IOException {
+      File outputFile = new File(file);
+      PrintStream ps = null;
+      try {
+         ps = new PrintStream(new FileOutputStream(outputFile));
+         ps.println(text);
+         return outputFile;
+      } catch (Exception e) {
+         throw new IOException("Failed to write text file '" + file + "' : "
+               + e.getMessage(), e);
+      } finally {
+         try {
+            ps.close();
+         } catch (Exception ignore) {
+         }
+      }
+   }
+}

Propchange: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/TextConfigFileWriter.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Copied: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java (from r1351904, oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java)
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java?p2=oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java&p1=oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java&r1=1351904&r2=1351905&rev=1351905&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/xslt/XslTransformWriter.java (original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java Wed Jun 20 00:23:59 2012
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.oodt.cas.pge.writers.xslt;
+package org.apache.oodt.cas.pge.writers;
 
 //JDK imports
 import java.io.File;
@@ -33,9 +33,8 @@ import javax.xml.transform.dom.DOMSource
 //OODT imports
 import org.apache.oodt.cas.metadata.Metadata;
 import org.apache.oodt.cas.metadata.SerializableMetadata;
-import org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter;
 
-/** 
+/**
  * XSL Transformation class which writes Science PGE config files based from the
  * XML format of SerializableMetadata.
  *
@@ -43,32 +42,32 @@ import org.apache.oodt.cas.pge.writers.S
  */
 public class XslTransformWriter implements SciPgeConfigFileWriter {
 
-    public File createConfigFile(String sciPgeConfigFilePath,
-            Metadata inputMetadata, Object... customArgs) throws IOException {
-        try {
-            File sciPgeConfigFile = new File(sciPgeConfigFilePath);
-
-            String xsltFilePath = (String) customArgs[0];
-            Source xsltSource = new StreamSource(new File(xsltFilePath));
-            Result result = new StreamResult(sciPgeConfigFile);
-
-            TransformerFactory transFact = TransformerFactory.newInstance();
-            Transformer trans = transFact.newTransformer(xsltSource);
-            boolean useCDATA = customArgs.length > 1 ? ((String) customArgs[1])
-                    .toLowerCase().equals("true") : false;
-            Source xmlSource = new DOMSource((new SerializableMetadata(
-                    inputMetadata,
-                    trans.getOutputProperty(OutputKeys.ENCODING), useCDATA))
-                    .toXML());
-
-            trans.setOutputProperty(OutputKeys.INDENT, "yes");
-            trans.transform(xmlSource, result);
-
-            return sciPgeConfigFile;
-        } catch (Exception e) {
-            e.printStackTrace();
-            throw new IOException("Failed to create science PGE config file '"
-                    + sciPgeConfigFilePath + "' : " + e.getMessage());
-        }
-    }
+   @Override
+   public File createConfigFile(String sciPgeConfigFilePath,
+         Metadata inputMetadata, Object... customArgs) throws IOException {
+      try {
+         File sciPgeConfigFile = new File(sciPgeConfigFilePath);
+
+         String xsltFilePath = (String) customArgs[0];
+         Source xsltSource = new StreamSource(new File(xsltFilePath));
+         Result result = new StreamResult(sciPgeConfigFile);
+
+         TransformerFactory transFact = TransformerFactory.newInstance();
+         Transformer trans = transFact.newTransformer(xsltSource);
+         boolean useCDATA = customArgs.length > 1 ? ((String) customArgs[1])
+               .toLowerCase().equals("true") : false;
+         Source xmlSource = new DOMSource((new SerializableMetadata(
+               inputMetadata, trans.getOutputProperty(OutputKeys.ENCODING),
+               useCDATA)).toXML());
+
+         trans.setOutputProperty(OutputKeys.INDENT, "yes");
+         trans.transform(xmlSource, result);
+
+         return sciPgeConfigFile;
+      } catch (Exception e) {
+         e.printStackTrace();
+         throw new IOException("Failed to create science PGE config file '"
+               + sciPgeConfigFilePath + "' : " + e.getMessage());
+      }
+   }
 }

Propchange: oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/writers/XslTransformWriter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: oodt/trunk/pge/src/main/testdata/blank-pge-config.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/blank-pge-config.xml?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/blank-pge-config.xml (added)
+++ oodt/trunk/pge/src/main/testdata/blank-pge-config.xml Wed Jun 20 00:23:59 2012
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Empty PgeConfig file for testing purposes.
+-->
+<pgeConfig>    
+</pgeConfig>

Propchange: oodt/trunk/pge/src/main/testdata/blank-pge-config.xml
------------------------------------------------------------------------------
    svn:executable = *

Propchange: oodt/trunk/pge/src/main/testdata/blank-pge-config.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml (added)
+++ oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml Wed Jun 20 00:23:59 2012
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Only customMetadata other PgeConfig XML file for testing purposes.
+-->
+<pgeConfig>
+
+	<customMetadata>
+		<metadata key="GreaterThan" val="&gt;" />
+		<metadata key="LessThan" val="&lt;" />
+    <metadata key="Ampersand" val="&amp;" />
+    <metadata key="Apostrophe" val="&apos;" />
+    <metadata key="QuotationMark" val="&quot;" />		
+	</customMetadata>
+
+</pgeConfig>

Propchange: oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml
------------------------------------------------------------------------------
    svn:executable = *

Propchange: oodt/trunk/pge/src/main/testdata/common-metadata-pge-config.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml (added)
+++ oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml Wed Jun 20 00:23:59 2012
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Only customMetadata other PgeConfig XML file for testing purposes.
+-->
+<pgeConfig>
+
+	<customMetadata>
+		<metadata key="" val="" />
+		<metadata key="" key-ref="" />
+		<metadata key-gen="" val="" />
+	</customMetadata>
+
+</pgeConfig>

Propchange: oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml
------------------------------------------------------------------------------
    svn:executable = *

Propchange: oodt/trunk/pge/src/main/testdata/custom-metadata-only-pge-config.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/data-file-1.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/data-file-1.txt?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/data-file-1.txt (added)
+++ oodt/trunk/pge/src/main/testdata/data-file-1.txt Wed Jun 20 00:23:59 2012
@@ -0,0 +1 @@
+This is a test data file 1
\ No newline at end of file

Propchange: oodt/trunk/pge/src/main/testdata/data-file-1.txt
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/data-file-2.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/data-file-2.txt?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/data-file-2.txt (added)
+++ oodt/trunk/pge/src/main/testdata/data-file-2.txt Wed Jun 20 00:23:59 2012
@@ -0,0 +1 @@
+This is a test data file 2
\ No newline at end of file

Propchange: oodt/trunk/pge/src/main/testdata/data-file-2.txt
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml (added)
+++ oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml Wed Jun 20 00:23:59 2012
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pgeConfig>
+
+	<customMetadata>
+	  <metadata key="TextWriter" val="org.apache.oodt.cas.pge.writers.TextConfigFileWriter" />
+	  <metadata key="CsvWriter" val="org.apache.oodt.cas.pge.writers.CsvConfigFileWriter" />
+	</customMetadata>
+
+</pgeConfig>

Propchange: oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml
------------------------------------------------------------------------------
    svn:executable = *

Propchange: oodt/trunk/pge/src/main/testdata/dyn-input-writers-pge-config.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml (added)
+++ oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml Wed Jun 20 00:23:59 2012
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Only imports other PgeConfig XML file for testing purposes.
+-->
+<pgeConfig>
+    
+    <import namespace="blank" file="blank-pge-config.xml"/>
+    <import file="pge-config.xml"/>
+
+</pgeConfig>

Propchange: oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml
------------------------------------------------------------------------------
    svn:executable = *

Propchange: oodt/trunk/pge/src/main/testdata/imports-only-pge-config.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/main/testdata/pge-config.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/testdata/pge-config.xml?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/main/testdata/pge-config.xml (added)
+++ oodt/trunk/pge/src/main/testdata/pge-config.xml Wed Jun 20 00:23:59 2012
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more contributor
+license agreements.  See the NOTICE.txt file distributed with this work for
+additional information regarding copyright ownership.  The ASF licenses this
+file to you under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License.  You may obtain a copy of
+the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+License for the specific language governing permissions and limitations under
+the License.
+
+TODO(bfoster): Make this a working example.
+-->
+<pgeConfig>
+
+  <import namespace="commons" file="common-metadata-pge-config.xml" />
+  <import namespace="writers" file="dyn-input-writers-pge-config.xml" />
+
+	<dynInputFiles>
+		<file path="[TextInputFile]" writerClass="[writers/TextWriter]" args="[TextFileTemplate]" />
+    <file path="[CsvInputFile]" writerClass="[writers/CsvWriter]" args="[CsvHeader]" />
+	</dynInputFiles>
+
+  <fileStaging dir="[StagingDir]" force="[ForceStaging]">
+    <stageFiles metadataKey="InputFiles" />
+  </fileStaging>
+
+	<exe dir="[WorkingDir]" shellType="csh">
+		<cmd>echo [TextInputFile] [commons/GreaterThan] [OutputFileName]</cmd>
+    <cmd>echo [CsvInputFile] [commons/GreaterThan][commons/GreaterThan] [OutputFileName]</cmd>
+    <cmd>if ( ! -e [InputFile1] || ! -e [InputFile2] ) then</cmd>
+    <cmd>  exit 1</cmd>
+    <cmd>endif</cmd>
+	</exe>
+
+	<output>
+		<dir path="[OutputDir]" createBeforeExe="true"/>
+	</output>
+
+	<customMetadata>
+    <!-- Defaults -->
+    <metadata key="GREETING_ENUM" val="Simple" />
+    <metadata key="FORCE_STAGING" val="true" />
+    <metadata key="CUSTOM_GREETING_ENUM" val="Custom" />
+    <metadata key="CUSTOM_GREETING_ENUM_VALUE" val="[commons/LessThan]Custom Greeting Here[commons/GreaterThan]" />
+    
+	  <!-- Remap externally set metadata so if name changes easy to update config file -->
+	  <metadata key="WorkingDir" key-ref="WORKING_DIR" />
+    <metadata key="GreetingEnum" key-ref="GREETING_ENUM" />
+    <metadata key="CustomGreetingEnum" key-ref="CUSTOM_GREETING_ENUM" />
+    <metadata key="CustomGreetingEnumValue" key-ref="CUSTOM_GREETING_ENUM_VALUE" />
+    <metadata key="InputFile1" key-ref="INPUT_FILE_1" />
+    <metadata key="InputFile2" key-ref="INPUT_FILE_2" />
+    <metadata key="ForceStaging" key-ref="FORCE_STAGING" />
+
+    <!-- Input Files -->
+    <metadata key="InputFiles" val="[InputFile1],[InputFile2]" split="true" />
+    
+    <!-- Working sub-directories -->
+    <metadata key="ConfigDir" val="[WorkingDir]/config" />
+    <metadata key="OutputDir" val="[WorkingDir]/output" />
+    <metadata key="StagingDir" val="[WorkingDir]/staging" />
+
+    <!-- Dynamically generated PGE input config files -->
+    <metadata key="DynInput/Text/Name" val="dyn-input.txt" />
+    <metadata key="DynInput/CSV/Name" val="dyn-input.csv" />
+    <metadata key="TextInputFile" val="[ConfigDir]/[DynInput/Text/Name]" />
+    <metadata key="CsvInputFile" val="[ConfigDir]/[DynInput/CSV/Name]" />
+
+    <!-- Output file names -->
+    <metadata key="OutputFileName" val="PgeOutput.txt" />
+
+    <!-- Text input file template metadata -->
+    <metadata key="TextFileTemplate" envReplace="false">
+      [Greeting],
+      This is a template for text file [TextInputFile].
+      This template was written at: [DATE.UTC].
+      [Signature]
+    </metadata>
+    <metadata key="SimpleGreeting" val="Hello" />
+    <metadata key="CasualGreeting" val="Hi" />    
+    <metadata key-gen="[CustomGreetingEnum]Greeting" val="[CustomGreetingEnumValue]" />
+    <metadata key="Greating" key-ref="[GreetingEnum]Greeting" />
+    <metadata key="Signature" val="-bfost" />
+
+    <!-- CSV input file metadata -->
+    <metadata key="AuxInputFiles" val="File1.txt,File2.dat,File3.xml" split="true" />
+    <metadata key="IsText" val="true,false,true" split="true" />
+    <metadata key="CsvHeader" val="AuxInputFiles,IsText" split="true" />
+	</customMetadata>
+
+</pgeConfig>

Propchange: oodt/trunk/pge/src/main/testdata/pge-config.xml
------------------------------------------------------------------------------
    svn:executable = *

Propchange: oodt/trunk/pge/src/main/testdata/pge-config.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/TestPGETaskInstance.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/TestPGETaskInstance.java?rev=1351905&r1=1351904&r2=1351905&view=diff
==============================================================================
--- oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/TestPGETaskInstance.java (original)
+++ oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/TestPGETaskInstance.java Wed Jun 20 00:23:59 2012
@@ -83,13 +83,14 @@ import com.google.common.collect.Sets;
 
 /**
  * Test class for {@link PGETaskInstance}.
- * 
+ *
  * @author bfoster (Brian Foster)
  */
 public class TestPGETaskInstance extends TestCase {
 
-   private List<File> tmpDirs = Lists.newArrayList();
+   private final List<File> tmpDirs = Lists.newArrayList();
 
+   @Override
    public void tearDown() throws Exception {
       for (File tmpDir : tmpDirs) {
          FileUtils.forceDelete(tmpDir);
@@ -199,7 +200,7 @@ public class TestPGETaskInstance extends
             @Override
             public boolean accept(File pathname) {
                return pathname.getName().endsWith(".log");
-            } 
+            }
          })[0], "UTF-8");
       assertEquals("INFO: pge1 message1", messages.get(1));
       assertEquals("INFO: pge1 message2", messages.get(3));
@@ -211,7 +212,7 @@ public class TestPGETaskInstance extends
             @Override
             public boolean accept(File pathname) {
                return pathname.getName().endsWith(".log");
-            } 
+            }
          })[0], "UTF-8");
       assertEquals("SEVERE: pge2 message1", messages.get(1));
    }
@@ -220,6 +221,7 @@ public class TestPGETaskInstance extends
       final Map<String, String> args = Maps.newHashMap();
       PGETaskInstance pgeTask = createTestInstance();
       pgeTask.wm = new XmlRpcWorkflowManagerClient(null) {
+         @Override
          public boolean updateWorkflowInstanceStatus(String instanceId,
                String status) {
             args.put("InstanceId", instanceId);
@@ -239,7 +241,7 @@ public class TestPGETaskInstance extends
       final String KEY = "TestKey";
       final String VALUE = "TestValue";
       File pgeConfigFile = new File(createTmpDir(), "pgeConfig.xml");
-      FileUtils.writeLines(pgeConfigFile, "UTF-8", 
+      FileUtils.writeLines(pgeConfigFile, "UTF-8",
             Lists.newArrayList(
                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
                "<pgeConfig>",
@@ -394,7 +396,7 @@ public class TestPGETaskInstance extends
    }
 
    public void testRunIngestCrawler() throws Exception {
-      // Case: UpdateStatus Success, VerifyIngest Success, 
+      // Case: UpdateStatus Success, VerifyIngest Success,
       PGETaskInstance pgeTask = createTestInstance();
       pgeTask.pgeConfig.addOuputDirAndExpressions(new OutputDir("/tmp/dir1", true));
       pgeTask.pgeConfig.addOuputDirAndExpressions(new OutputDir("/tmp/dir2", true));

Added: oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/config/TestXmlFilePgeConfigBuilder.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/config/TestXmlFilePgeConfigBuilder.java?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/config/TestXmlFilePgeConfigBuilder.java (added)
+++ oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/config/TestXmlFilePgeConfigBuilder.java Wed Jun 20 00:23:59 2012
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.pge.config;
+
+//OODT static imports
+import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.CONFIG_FILE_PATH;
+
+//JDK imports
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+//Google imports
+import com.google.common.collect.Lists;
+
+//OODT imports
+import org.apache.oodt.cas.pge.metadata.PgeMetadata;
+
+//JUnit imports
+import junit.framework.TestCase;
+
+/**
+ * Test class for {@link XmlFilePgeConfigBuilder}.
+ *
+ * @author bfoster (Brian Foster)
+ */
+public class TestXmlFilePgeConfigBuilder extends TestCase {
+
+   public static final String CONFIG_FILE = "src/main/testdata/pge-config.xml";
+
+   public void testBuild() throws IOException {
+      XmlFilePgeConfigBuilder builder = new XmlFilePgeConfigBuilder();
+      PgeMetadata pgeMetadata = new PgeMetadata();
+      pgeMetadata.replaceMetadata(CONFIG_FILE_PATH, CONFIG_FILE);
+      pgeMetadata.replaceMetadata("INPUT_FILE_1", "src/main/testdata/data-file-1.txt");
+      pgeMetadata.replaceMetadata("INPUT_FILE_2", "src/main/testdata/data-file-2.txt");
+      pgeMetadata.replaceMetadata("WORKING_DIR", "/tmp");
+      pgeMetadata.markAsDynamicMetadataKey();
+      pgeMetadata.commitMarkedDynamicMetadataKeys();
+      PgeConfig pgeConfig = builder.build(pgeMetadata);
+
+      // Verify metadata checks out.
+      assertEquals(40, pgeMetadata.asMetadata().getAllKeys().size());
+      assertEquals(CONFIG_FILE, pgeMetadata.getMetadata(CONFIG_FILE_PATH));
+      assertEquals("src/main/testdata/data-file-1.txt", pgeMetadata.getMetadata("INPUT_FILE_1"));
+      assertEquals("src/main/testdata/data-file-2.txt", pgeMetadata.getMetadata("INPUT_FILE_2"));
+      assertEquals("/tmp", pgeMetadata.getMetadata("WORKING_DIR"));
+      assertEquals(">", pgeMetadata.getMetadata("commons/GreaterThan"));
+      assertEquals("<", pgeMetadata.getMetadata("commons/LessThan"));
+      assertEquals("&", pgeMetadata.getMetadata("commons/Ampersand"));
+      assertEquals("'", pgeMetadata.getMetadata("commons/Apostrophe"));
+      assertEquals("\"", pgeMetadata.getMetadata("commons/QuotationMark"));
+      assertEquals("org.apache.oodt.cas.pge.writers.TextConfigFileWriter", pgeMetadata.getMetadata("writers/TextWriter"));
+      assertEquals("org.apache.oodt.cas.pge.writers.CsvConfigFileWriter", pgeMetadata.getMetadata("writers/CsvWriter"));
+      assertEquals("Simple", pgeMetadata.getMetadata("GREETING_ENUM"));
+      assertEquals("true", pgeMetadata.getMetadata("FORCE_STAGING"));
+      assertEquals("Custom", pgeMetadata.getMetadata("CUSTOM_GREETING_ENUM"));
+      assertEquals("<Custom Greeting Here>", pgeMetadata.getMetadata("CUSTOM_GREETING_ENUM_VALUE"));
+      assertEquals("/tmp", pgeMetadata.getMetadata("WorkingDir"));
+      assertEquals("Simple", pgeMetadata.getMetadata("GreetingEnum"));
+      assertEquals("Custom", pgeMetadata.getMetadata("CustomGreetingEnum"));
+      assertEquals("<Custom Greeting Here>", pgeMetadata.getMetadata("CustomGreetingEnumValue"));
+      assertEquals("src/main/testdata/data-file-1.txt", pgeMetadata.getMetadata("InputFile1"));
+      assertEquals("src/main/testdata/data-file-2.txt", pgeMetadata.getMetadata("InputFile2"));
+      assertEquals("true", pgeMetadata.getMetadata("ForceStaging"));
+      assertEquals(Lists.newArrayList("/tmp/staging/data-file-1.txt", "/tmp/staging/data-file-2.txt"), pgeMetadata.getAllMetadata("InputFiles"));
+      assertEquals("/tmp/config", pgeMetadata.getMetadata("ConfigDir"));
+      assertEquals("/tmp/output", pgeMetadata.getMetadata("OutputDir"));
+      assertEquals("/tmp/staging", pgeMetadata.getMetadata("StagingDir"));
+      assertEquals("dyn-input.txt", pgeMetadata.getMetadata("DynInput/Text/Name"));
+      assertEquals("dyn-input.csv", pgeMetadata.getMetadata("DynInput/CSV/Name"));
+      assertEquals("/tmp/config/dyn-input.txt", pgeMetadata.getMetadata("TextInputFile"));
+      assertEquals("/tmp/config/dyn-input.csv", pgeMetadata.getMetadata("CsvInputFile"));
+      assertEquals("PgeOutput.txt", pgeMetadata.getMetadata("OutputFileName"));
+      assertEquals("\n      [Greeting],\n      This is a template for text file [TextInputFile].\n      This template was written at: [DATE.UTC].\n      [Signature]\n    ", pgeMetadata.getMetadata("TextFileTemplate"));
+      assertEquals("Hello", pgeMetadata.getMetadata("SimpleGreeting"));
+      assertEquals("Hi", pgeMetadata.getMetadata("CasualGreeting"));
+      assertEquals("<Custom Greeting Here>", pgeMetadata.getMetadata("CustomGreeting"));
+      assertEquals("Hello", pgeMetadata.getMetadata("Greating"));
+      assertEquals("-bfost", pgeMetadata.getMetadata("Signature"));
+      assertEquals(Lists.newArrayList("File1.txt","File2.dat","File3.xml"), pgeMetadata.getAllMetadata("AuxInputFiles"));
+      assertEquals(Lists.newArrayList("true","false","true"), pgeMetadata.getAllMetadata("IsText"));
+      assertEquals(Lists.newArrayList("AuxInputFiles","IsText"), pgeMetadata.getAllMetadata("CsvHeader"));
+
+      // Verify pgeconfig checks out.
+      List<DynamicConfigFile> dynConfigFiles = pgeConfig.getDynamicConfigFiles();
+      assertEquals(2, dynConfigFiles.size());
+      assertEquals("org.apache.oodt.cas.pge.writers.TextConfigFileWriter", dynConfigFiles.get(0).getWriterClass());
+      assertEquals("/tmp/config/dyn-input.txt", dynConfigFiles.get(0).getFilePath());
+      assertEquals(Lists.newArrayList("\n      [Greeting],\n      This is a template for text file [TextInputFile].\n      This template was written at: [DATE.UTC].\n      [Signature]\n    "), Arrays.asList(dynConfigFiles.get(0).getArgs()));
+      assertEquals("org.apache.oodt.cas.pge.writers.CsvConfigFileWriter", dynConfigFiles.get(1).getWriterClass());
+      assertEquals("/tmp/config/dyn-input.csv", dynConfigFiles.get(1).getFilePath());
+      assertEquals(Lists.newArrayList("AuxInputFiles,IsText"), Arrays.asList(dynConfigFiles.get(1).getArgs()));
+
+      assertEquals(5, pgeConfig.getExeCmds().size());
+      assertEquals("echo /tmp/config/dyn-input.txt > PgeOutput.txt", pgeConfig.getExeCmds().get(0));
+      assertEquals("echo /tmp/config/dyn-input.csv >> PgeOutput.txt", pgeConfig.getExeCmds().get(1));
+      assertEquals("if ( ! -e src/main/testdata/data-file-1.txt || ! -e src/main/testdata/data-file-2.txt ) then", pgeConfig.getExeCmds().get(2));
+      assertEquals("  exit 1", pgeConfig.getExeCmds().get(3));
+      assertEquals("endif", pgeConfig.getExeCmds().get(4));
+      assertEquals("/tmp", pgeConfig.getExeDir());
+      assertEquals("csh", pgeConfig.getShellType());
+
+      FileStagingInfo fileStagingInfo = pgeConfig.getFileStagingInfo();
+      assertEquals(Lists.newArrayList("src/main/testdata/data-file-1.txt", "src/main/testdata/data-file-2.txt"), fileStagingInfo.getFilePaths());
+      assertEquals("/tmp/staging", fileStagingInfo.getStagingDir());
+      assertEquals(true, fileStagingInfo.isForceStaging());
+
+      List<OutputDir> outputDirs = pgeConfig.getOuputDirs();
+      assertEquals(1, outputDirs.size());
+      assertEquals(true, outputDirs.get(0).isCreateBeforeExe());
+      assertEquals("/tmp/output", outputDirs.get(0).getPath());
+   }
+}

Propchange: oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/config/TestXmlFilePgeConfigBuilder.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/util/TestXmlHelper.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/util/TestXmlHelper.java?rev=1351905&view=auto
==============================================================================
--- oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/util/TestXmlHelper.java (added)
+++ oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/util/TestXmlHelper.java Wed Jun 20 00:23:59 2012
@@ -0,0 +1,54 @@
+package org.apache.oodt.cas.pge.util;
+
+import static org.apache.oodt.cas.pge.config.PgeConfigMetKeys.IMPORT_TAG;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getFile;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getImports;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getNamespace;
+import static org.apache.oodt.cas.pge.util.XmlHelper.getRootElement;
+
+import java.io.FileNotFoundException;
+import java.util.List;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.apache.oodt.cas.metadata.Metadata;
+
+import junit.framework.TestCase;
+
+public class TestXmlHelper extends TestCase {
+
+   private static final String BLANK_PGE_CONFIG = "src/main/testdata/blank-pge-config.xml";
+   private static final String IMPORTS_ONLY_PGE_CONFIG = "src/main/testdata/imports-only-pge-config.xml";
+
+   // Tests that only root node is required.
+   public void testLoadBlankXmlFile() throws FileNotFoundException {
+      Element elem = getRootElement(BLANK_PGE_CONFIG);
+      assertNotNull(elem);
+      assertEquals(1, elem.getChildNodes().getLength());
+      assertEquals("#text", elem.getChildNodes().item(0).getNodeName());
+      assertEquals("", elem.getChildNodes().item(0).getNodeValue().trim());
+   }
+
+   public void testGetImports() throws Exception {
+      Element elem = getRootElement(IMPORTS_ONLY_PGE_CONFIG);
+      Metadata metadata = new Metadata();
+      List<Pair<String, String>> imports = getImports(elem, metadata);
+      assertEquals(2, imports.size());
+      assertEquals("blank", imports.get(0).getFirst());
+      assertEquals("blank-pge-config.xml", imports.get(0).getSecond());
+      assertEquals(null, imports.get(1).getFirst());
+      assertEquals("pge-config.xml", imports.get(1).getSecond());
+   }
+
+   public void testGetNamespace() throws Exception {
+      Element elem = getRootElement(IMPORTS_ONLY_PGE_CONFIG);
+      Metadata metadata = new Metadata();
+
+      NodeList importTags = elem.getElementsByTagName(IMPORT_TAG);
+      assertEquals("blank", getNamespace((Element) importTags.item(0), metadata));
+      assertEquals("blank-pge-config.xml", getFile((Element) importTags.item(0), metadata));
+      assertNull(getNamespace((Element) importTags.item(1), metadata));
+      assertEquals("pge-config.xml", getFile((Element) importTags.item(1), metadata));
+   }
+}

Propchange: oodt/trunk/pge/src/test/org/apache/oodt/cas/pge/util/TestXmlHelper.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message