Mailing-List: contact commits-help@oodt.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@oodt.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1052148 [6/17] - in
 /oodt/branches/wengine-branch/filemgr: ./
 .settings/ src/ src/main/ src/main/assembly/ src/main/bin/ src/main/java/
 src/main/java/gov/ src/main/java/gov/nasa/ src/main/java/gov/nasa/jpl/
 src/main/java/gov/nasa/jpl/oodt/...
Date: Thu, 23 Dec 2010 02:48:11 -0000
To: commits@oodt.apache.org
From: bfoster@apache.org
Message-Id: <20101223024814.354412388C1C@eris.apache.org>

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+//APACHE Imports
+import org.apache.commons.io.FileUtils;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
+import org.apache.oodt.cas.filemgr.structs.mime.MimeTypes;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
+
+//JDK imports
+import java.io.IOException;
+import java.io.File;
+import java.util.List;
+import java.util.Iterator;
+import java.util.Vector;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.net.URI;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.net.URISyntaxException;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * An implementation of the {@link DataTransfer} interface that moves products
+ * that are available via URIs on the same machine, through an NFS mounted disk,
+ * or via the locally mounted file repository.
+ * </p>
+ * 
+ */
+public class LocalDataTransferer implements DataTransfer {
+
+    /* our log stream */
+    private static Logger LOG = Logger.getLogger(LocalDataTransferer.class
+            .getName());
+
+    /* file manager client */
+    private XmlRpcFileManagerClient client = null;
+
+    /**
+     * <p>
+     * Default Constructor
+     * </p>
+     */
+    public LocalDataTransferer() {
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl(java.net.URL)
+     */
+    public void setFileManagerUrl(URL url) {
+        try {
+            client = new XmlRpcFileManagerClient(url);
+            LOG.log(Level.INFO, "Local Data Transfer to: ["
+                    + client.getFileManagerUrl().toString() + "] enabled");
+        } catch (ConnectionException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.datatransfer.DataTransfer#transferProduct(gov.nasa.jpl.oodt.cas.data.structs.Product)
+     */
+    public void transferProduct(Product product) throws DataTransferException,
+            IOException {
+        // check whether or not it's a set of files, or it's actually a dir
+        // structure
+        if (product.getProductStructure()
+                .equals(Product.STRUCTURE_HIERARCHICAL)) {
+            try {
+                moveDirToProductRepo(product);
+            } catch (URISyntaxException e) {
+                LOG.log(Level.WARNING, "URI Syntax Exception when moving dir "
+                        + ((Reference) product.getProductReferences().get(0))
+                                .getOrigReference() + ": Message: "
+                        + e.getMessage());
+                throw new DataTransferException(e);
+            }
+        } else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
+            try {
+                moveFilesToProductRepo(product);
+            } catch (URISyntaxException e) {
+                LOG.log(Level.WARNING,
+                        "URI Syntax Exception when moving files: Message: "
+                                + e.getMessage());
+                throw new DataTransferException(e);
+            }
+        } else {
+            throw new DataTransferException(
+                    "Cannot transfer product on unknown ProductStructure: "
+                            + product.getProductStructure());
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see gov.nasa.jpl.oodt.cas.filemgr.datatransfer.DataTransfer#copyProduct(gov.nasa.jpl.oodt.cas.filemgr.structs.Product, java.io.File)
+     */
+	public void copyProduct(Product product, File directory) throws DataTransferException,
+			IOException {
+        // check whether or not it's a set of files, or it's actually a dir
+        // structure
+        if (product.getProductStructure()
+                .equals(Product.STRUCTURE_HIERARCHICAL)) {
+            try {
+            	copyDirToDir(product, directory);
+            } catch (URISyntaxException e) {
+                LOG.log(Level.WARNING, "URI Syntax Exception when moving dir "
+                        + ((Reference) product.getProductReferences().get(0))
+                                .getDataStoreReference() + ": Message: "
+                        + e.getMessage());
+                throw new DataTransferException(e);
+            }
+        } else if (product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
+            try {
+            	copyFilesToDir(product, directory);
+            } catch (URISyntaxException e) {
+                LOG.log(Level.WARNING,
+                        "URI Syntax Exception when moving files: Message: "
+                                + e.getMessage());
+                throw new DataTransferException(e);
+            }
+        } else {
+            throw new DataTransferException(
+                    "Cannot transfer product on unknown ProductStructure: "
+                            + product.getProductStructure());
+        }
+	}
+
+    /**
+     * @param args
+     */
+    public static void main(String[] args) throws DataTransferException,
+            IOException, URISyntaxException {
+        String usage = "LocalFileTransfer --productName <name> --productRepo <repo> [--dir <dirRef>] [--files <origRef 1>...<origRef N>]\n";
+
+        MimeTypes mimeTypeRepo = MimeTypes
+                .buildRepository(System
+                        .getProperty("gov.nasa.jpl.oodt.cas.filemgr.mime.type.repository"));
+
+        String productName = null;
+        String productRepo = null;
+        String transferType = null;
+        Reference dirReference = null;
+
+        List<Reference> fileReferences = null;
+
+        for (int i = 0; i < args.length; i++) {
+            if (args[i].equals("--dir")) {
+                transferType = "dir";
+                dirReference = new Reference();
+                dirReference.setOrigReference(new File(new URI(args[++i]))
+                        .toURI().toString());
+                LOG.log(Level.FINER,
+                        "LocalFileTransfer.main: Generated orig reference: "
+                                + dirReference.getOrigReference());
+            } else if (args[i].equals("--files")) {
+                transferType = "files";
+                fileReferences = new Vector<Reference>();
+                for (int j = i + 1; j < args.length; j++) {
+                    LOG.log(Level.FINER,
+                            "LocalFileTransfer.main: Adding file ref: "
+                                    + args[j]);
+                    fileReferences.add(new Reference(args[j], null, new File(
+                            args[j]).length(), mimeTypeRepo
+                            .getMimeType(args[j])));
+                }
+            } else if (args[i].equals("--productName")) {
+                productName = args[++i];
+            } else if (args[i].equals("--productRepo")) {
+                productRepo = args[++i];
+            }
+        }
+
+        if (transferType == null
+                || (transferType != null && ((transferType.equals("dir") && dirReference == null)
+                        || (transferType.equals("files") && fileReferences == null)
+                        || (transferType != null && !(transferType
+                                .equals("dir") || transferType.equals("files")))
+                        || productName == null || productRepo == null))) {
+            System.err.println(usage);
+            System.exit(1);
+        }
+
+        // construct a new Product
+        Product p = new Product();
+        p.setProductName(productName);
+
+        if (transferType.equals("dir")) {
+            p.setProductStructure(Product.STRUCTURE_HIERARCHICAL);
+            dirReference.setDataStoreReference(new File(new URI(productRepo))
+                    .toURL().toExternalForm()
+                    + URLEncoder.encode(p.getProductName(), "UTF-8") + "/");
+            p.getProductReferences().add(dirReference);
+            /* we'll do a simple versioning scheme ourselves: no versioning! */
+            p.getProductReferences().addAll(
+                    VersioningUtils.getReferencesFromDir(new File(new URI(
+                            dirReference.getOrigReference()))));
+            VersioningUtils.createBasicDataStoreRefsHierarchical(p
+                    .getProductReferences());
+        } else if (transferType.equals("files")) {
+            p.setProductStructure("Flat");
+            p.getProductReferences().addAll(fileReferences);
+            VersioningUtils.createBasicDataStoreRefsFlat(productName,
+                    productRepo, p.getProductReferences());
+        }
+
+        DataTransfer transfer = new LocalDataTransferer();
+        transfer.transferProduct(p);
+
+    }
+
+    private void moveDirToProductRepo(Product product) throws IOException,
+            URISyntaxException {
+        Reference dirRef = (Reference) product.getProductReferences().get(0);
+        LOG.log(Level.INFO, "LocalDataTransferer: Moving Directory: "
+                + dirRef.getOrigReference() + " to "
+                + dirRef.getDataStoreReference());
+
+        // notify the file manager that we started
+        quietNotifyTransferProduct(product);
+
+        for (Iterator<Reference> i = product.getProductReferences().iterator(); i
+                .hasNext();) {
+            Reference r = i.next();
+            File fileRef = new File(new URI(r.getOrigReference()));
+
+            if (fileRef.isFile()) {
+                moveFile(r, false);
+            } else if (fileRef.isDirectory()
+                    && (fileRef.list() != null && fileRef.list().length == 0)) {
+                // if it's a directory and it doesn't exist yet, we should
+                // create it
+                // just in case there's no files in it
+                if (!new File(new URI(r.getDataStoreReference())).exists()) {
+                    LOG.log(Level.FINER, "Directory: ["
+                            + r.getDataStoreReference()
+                            + "] doesn't exist: creating it");
+                    if (!new File(new URI(r.getDataStoreReference())).mkdirs()) {
+                        LOG.log(Level.WARNING, "Unable to create directory: ["
+                                + r.getDataStoreReference()
+                                + "] in local data transferer");
+                    }
+                }
+            }
+        }
+
+        // notify the file manager that we're done
+        quietNotifyProductTransferComplete(product);
+
+    }
+    
+    private void copyDirToDir(Product product, File directory) throws IOException,
+			URISyntaxException {
+		Reference dirRef = (Reference) product.getProductReferences().get(0);
+		LOG.log(Level.INFO, "LocalDataTransferer: Staging Directory: "
+				+ dirRef.getDataStoreReference() + " into directory "
+				+ directory.getAbsolutePath());
+
+		for (Iterator<Reference> i = product.getProductReferences().iterator(); i
+				.hasNext();) {
+			Reference r = i.next();
+			File fileRef = new File(new URI(r.getDataStoreReference()));
+
+			if (fileRef.isFile()) {
+				copyFile(r, directory);
+			} else if (fileRef.isDirectory()
+					&& (fileRef.list() != null && fileRef.list().length == 0)) {
+				// if it's a directory and it doesn't exist yet, we should
+				// create it
+				// just in case there's no files in it
+				File dest = new File(directory, fileRef.getName());
+				if (!new File(new URI(dest.getAbsolutePath())).exists()) {
+					LOG.log(Level.FINER, "Directory: ["
+							+ dest.getAbsolutePath()
+							+ "] doesn't exist: creating it");
+					if (!new File(new URI(dest.getAbsolutePath())).mkdirs()) {
+						LOG.log(Level.WARNING, "Unable to create directory: ["
+								+ dest.getAbsolutePath()
+								+ "] in local data transferer");
+					}
+				}
+			}
+		}
+	}
+
+    private void moveFilesToProductRepo(Product product) throws IOException,
+            URISyntaxException {
+        List<Reference> refs = product.getProductReferences();
+
+        // notify the file manager that we started
+        quietNotifyTransferProduct(product);
+
+        for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
+            Reference r = (Reference) i.next();
+            moveFile(r, true);
+        }
+
+        // notify the file manager that we're done
+        quietNotifyProductTransferComplete(product);
+    }
+    
+    private void copyFilesToDir(Product product, File directory) throws IOException,
+			URISyntaxException {
+		List<Reference> refs = product.getProductReferences();
+		for (Iterator<Reference> i = refs.iterator(); i.hasNext();) {
+			Reference r = (Reference) i.next();
+			copyFile(r, directory);
+		}
+	}
+
+    private void moveFile(Reference r, boolean log) throws IOException,
+            URISyntaxException {
+        if (log) {
+            LOG
+                    .log(Level.INFO, "LocalDataTransfer: Moving File: "
+                            + r.getOrigReference() + " to "
+                            + r.getDataStoreReference());
+        }
+        File srcFileRef = new File(new URI(r.getOrigReference()));
+        File destFileRef = new File(new URI(r.getDataStoreReference()));
+
+        FileUtils.copyFile(srcFileRef, destFileRef);
+    }
+    
+    private void copyFile(Reference r, File directory) throws IOException,
+			URISyntaxException {
+		File srcFileRef = new File(new URI(r.getDataStoreReference()));
+		LOG.log(Level.INFO, "LocalDataTransfer: Copying File: "
+				+ r.getDataStoreReference() + " to file:"
+				+ directory.getAbsolutePath() + "/" + srcFileRef.getName());
+		FileUtils.copyFile(srcFileRef, new File(directory, srcFileRef.getName()));
+	}
+
+    private void quietNotifyTransferProduct(Product p) {
+        if (client == null) {
+            LOG
+                    .log(Level.WARNING,
+                            "File Manager service not defined: this transfer will not be tracked");
+            return;
+        }
+
+        try {
+            client.transferringProduct(p);
+        } catch (DataTransferException e) {
+            e.printStackTrace();
+            LOG.log(Level.WARNING,
+                    "Error notifying file manager of product transfer initiation for product: ["
+                            + p.getProductId() + "]: Message: "
+                            + e.getMessage());
+            return;
+        }
+    }
+
+    private void quietNotifyProductTransferComplete(Product p) {
+        if (client == null) {
+            LOG
+                    .log(Level.WARNING,
+                            "File Manager service not defined: this transfer will not be tracked");
+            return;
+        }
+
+        try {
+            client.removeProductTransferStatus(p);
+        } catch (DataTransferException e) {
+            e.printStackTrace();
+            LOG.log(Level.WARNING,
+                    "Error notifying file manager of product transfer completion for product: ["
+                            + p.getProductId() + "]: Message: "
+                            + e.getMessage());
+            return;
+        }
+    }
+
+}
\ No newline at end of file

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferFactory.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferFactory.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferFactory.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferFactory.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+//JDK imports
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author mattmann
+ * @version $Revision$
+ * 
+ * <p>
+ * A Factory for constructing {@link RemoteDataTransferer} objects.
+ * </p>
+ * 
+ */
+public class RemoteDataTransferFactory implements DataTransferFactory {
+
+    /* the chunk size for sending files remotely */
+    private int chunkSize = 0;
+
+    /* our log stream */
+    private static Logger LOG = Logger
+            .getLogger(RemoteDataTransferFactory.class.getName());
+
+    /**
+     * 
+     */
+    public RemoteDataTransferFactory() throws InstantiationException {
+        chunkSize = Integer.getInteger(
+                "gov.nasa.jpl.oodt.cas.filemgr.datatransfer.remote.chunkSize",
+                1024).intValue();
+
+        LOG.log(Level.INFO, "RemoteDataTransfer enabled: using chunk size: ["
+                + chunkSize + "]");
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.datatransfer.DataTransferFactory#createDataTransfer()
+     */
+    public RemoteDataTransferer createDataTransfer() {
+        return new RemoteDataTransferer(chunkSize);
+    }
+
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+//JDK imports
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * An implementation of the {@link DataTransfer} interface that transfers files
+ * to a remote file manager over XML-RPC, using the File Manager Client.
+ * </p>
+ * 
+ */
+public class RemoteDataTransferer implements DataTransfer {
+
+    /*
+     * the url pointer to the file manager that we'll remotely transfer the file
+     * to
+     */
+    private URL fileManagerUrl = null;
+
+    /*
+     * the size of the chunks that files should be transferred over XML-RPC
+     * using
+     */
+    private int chunkSize = 1024;
+
+    /* our file manager client */
+    private XmlRpcFileManagerClient client = null;
+
+    /* our log stream */
+    private static Logger LOG = Logger.getLogger(RemoteDataTransferer.class
+            .getName());
+
+    /**
+     * 
+     */
+    public RemoteDataTransferer(int chunkSz) {
+        this.chunkSize = chunkSz;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.datatransfer.DataTransfer#setFileManagerUrl(java.net.URL)
+     */
+    public void setFileManagerUrl(URL url) {
+        try {
+            client = new XmlRpcFileManagerClient(url);
+            this.fileManagerUrl = url;
+            LOG.log(Level.INFO, "Remote Data Transfer to: ["
+                    + client.getFileManagerUrl().toString() + "] enabled");
+        } catch (ConnectionException e) {
+            LOG.log(Level.WARNING, "Connection exception for filemgr: [" + url
+                    + "]");
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.datatransfer.DataTransfer#transferProduct(gov.nasa.jpl.oodt.cas.filemgr.structs.Product)
+     */
+    public void transferProduct(Product product) throws DataTransferException,
+            IOException {
+
+        if (fileManagerUrl == null) {
+            throw new DataTransferException(
+                    "No file manager url specified for remote data transfer: cannot transfer product: ["
+                            + product.getProductName() + "]!");
+        }
+
+        quietNotifyTransferProduct(product);
+
+        // for each file reference, transfer the file to the remote file manager
+        for (Iterator<Reference> i = product.getProductReferences().iterator(); i
+                .hasNext();) {
+            Reference r = i.next();
+            // test whether or not the reference is a directory or a file
+            File refFile = null;
+            try {
+                refFile = new File(new URI(r.getOrigReference()));
+            } catch (URISyntaxException e) {
+                LOG.log(Level.WARNING, "Unable to test if reference: ["
+                        + r.getOrigReference()
+                        + "] is a directory: skipping it");
+                continue;
+            }
+
+            if (!refFile.isDirectory()) {
+                LOG.log(Level.FINE, "Reference: [" + r.getOrigReference()
+                        + "] is file: transferring it");
+
+                try {
+                    remoteTransfer(r, product);
+                } catch (URISyntaxException e) {
+                    LOG.log(Level.WARNING, "Error transferring file: ["
+                            + r.getOrigReference() + "]: URISyntaxException: "
+                            + e.getMessage());
+                }
+            } else {
+                LOG.log(Level.FINE, "RemoteTransfer: skipping reference: ["
+                        + refFile.getAbsolutePath() + "] of product: ["
+                        + product.getProductName() + "]: ref is a directory");
+            }
+        }
+
+        quietNotifyProductTransferComplete(product);
+
+    }
+    
+    /*
+     * (non-Javadoc)
+     * @see gov.nasa.jpl.oodt.cas.filemgr.datatransfer.DataTransfer#copyProduct(gov.nasa.jpl.oodt.cas.filemgr.structs.Product, java.io.File)
+     */
+	public void copyProduct(Product product, File directory) throws DataTransferException, IOException {
+		for (Reference reference : product.getProductReferences()) {
+			FileOutputStream fOut = null;
+	    	try {
+	    		File dataStoreFile = new File(new URI(reference.getDataStoreReference()));
+	    		File dest = new File(directory, dataStoreFile.getName());
+	    		fOut = new FileOutputStream(dest, false);
+	    		LOG.log(Level.INFO, "RemoteDataTransfer: Copying File: "
+	    				+ "fmp:" + dataStoreFile.getAbsolutePath() + " to "
+	    				+ "file:" + dest.getAbsolutePath());
+	    		byte[] fileData = null;
+		    	int offset = 0;
+		    	while(true) {
+		            fileData = (byte[]) client.retrieveFile(dataStoreFile.getAbsolutePath(), offset, 1024);
+		            if (fileData.length <= 0)
+		            	break;
+	                fOut.write(fileData);
+	                if (fileData.length < 1024)
+	                	break;
+		            offset += 1024;
+		    	}
+	    	}catch(Exception e) {
+	    		throw new DataTransferException("", e);
+	    	}finally {
+	    		try {
+	    			fOut.close();
+	    		}catch (Exception e) {}
+	    	}
+		}	
+	}
+
+    private void remoteTransfer(Reference reference, Product product)
+            throws URISyntaxException {
+        // get the file path
+        File origFile = new File(new URI(reference.getOrigReference()));
+        File destFile = new File(new URI(reference.getDataStoreReference()));
+        String origFilePath = origFile.getAbsolutePath();
+        String destFilePath = destFile.getAbsolutePath();
+
+        // read the file in chunk by chunk
+
+        byte[] buf = new byte[chunkSize];
+
+        FileInputStream is = null;
+
+        try {
+            is = new FileInputStream(origFile);
+            int offset = 0;
+            int numBytes = 0;
+
+            // remove the file if it already exists: this operation
+            // is an overwrite
+            if (!client.removeFile(destFilePath)) {
+                LOG.log(Level.WARNING,
+                        "RemoteDataTransfer: attempt to perform overwrite of dest file: ["
+                                + destFilePath + "] failed");
+            }
+
+            while ((numBytes = is.read(buf, offset, chunkSize)) != -1) {
+                client.transferFile(destFilePath, buf, offset, numBytes);
+            }
+        } catch (IOException e) {
+            LOG.log(Level.WARNING,
+                    "Error opening input stream to read file to transfer: Message: "
+                            + e.getMessage());
+            return;
+        } catch (DataTransferException e) {
+            LOG.log(Level.WARNING,
+                    "DataTransferException when transfering file: ["
+                            + origFilePath + "] to [" + destFilePath
+                            + "]: Message: " + e.getMessage());
+            return;
+        } finally {
+            if (is != null) {
+                try {
+                    is.close();
+                } catch (Exception ignore) {
+                }
+
+                is = null;
+            }
+        }
+    }
+
+    private void quietNotifyTransferProduct(Product p) {
+        try {
+            client.transferringProduct(p);
+        } catch (DataTransferException e) {
+            e.printStackTrace();
+            LOG.log(Level.WARNING,
+                    "Error notifying file manager of product transfer initiation for product: ["
+                            + p.getProductId() + "]: Message: "
+                            + e.getMessage());
+            return;
+        }
+    }
+
+    private void quietNotifyProductTransferComplete(Product p) {
+        try {
+            client.removeProductTransferStatus(p);
+        } catch (DataTransferException e) {
+            e.printStackTrace();
+            LOG.log(Level.WARNING,
+                    "Error notifying file manager of product transfer completion for product: ["
+                            + p.getProductId() + "]: Message: "
+                            + e.getMessage());
+            return;
+        }
+    }
+
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/TransferStatusTracker.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/TransferStatusTracker.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/TransferStatusTracker.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/TransferStatusTracker.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+//JDK imports
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.catalog.Catalog;
+import org.apache.oodt.cas.filemgr.structs.FileTransferStatus;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * An abstract base class for data transfers that uses an in-memory HashMap to
+ * keep track of data transfer status information.
+ * </p>
+ * 
+ */
+public class TransferStatusTracker {
+    /* HashMap containing a list of current product transfers */
+    protected HashMap<String, Product> currentProductTransfers = new HashMap<String, Product>();
+
+    /* our catalog object */
+    private Catalog catalog = null;
+
+    /* our log stream */
+    private static Logger LOG = Logger.getLogger(TransferStatusTracker.class
+            .getName());
+
+    /**
+     * <p>
+     * Default Constructor
+     * </p>.
+     * 
+     * @param catalog
+     *            The Catalog interface handed to it by the File Manager. It
+     *            needs this object to look up reference information.
+     */
+    public TransferStatusTracker(Catalog catalog) {
+        this.catalog = catalog;
+    }
+
+    public FileTransferStatus getCurrentFileTransfer() {
+        List<FileTransferStatus> transfers = getCurrentFileTransfers();
+
+        if (transfers != null && transfers.size() > 0) {
+            return (FileTransferStatus) transfers.get(0);
+        } else
+            return null;
+    }
+
+    public void transferringProduct(Product product) {
+        currentProductTransfers.put(product.getProductId(), product);
+    }
+
+    public List<FileTransferStatus> getCurrentFileTransfers() {
+        List<FileTransferStatus> currTransfers = new Vector<FileTransferStatus>();
+
+        for (Iterator<String> i = currentProductTransfers.keySet().iterator(); i
+                .hasNext();) {
+            String productId = i.next();
+            Product p = (Product) currentProductTransfers.get(productId);
+
+            // get its references
+            List<Reference> refs = quietGetReferences(p);
+
+            if (refs != null && refs.size() > 0) {
+                for (Iterator<Reference> j = refs.iterator(); j.hasNext();) {
+                    Reference r = j.next();
+
+                    long bytesTransferred = getBytesTransferred(r);
+
+                    if (bytesTransferred > 0
+                            && bytesTransferred < r.getFileSize() && !isDir(r)) {
+                        FileTransferStatus status = new FileTransferStatus();
+                        status.setBytesTransferred(bytesTransferred);
+                        status.setFileRef(r);
+                        status.setParentProduct(p);
+                        currTransfers.add(status);
+                    }
+                }
+            }
+        }
+
+        return currTransfers;
+    }
+
+    public double getPctTransferred(Product product) {
+        // get its references
+        List<Reference> refs = quietGetReferences(product);
+        long totalBytesTransferred = 0L;
+        long totalProductSize = 0L;
+
+        if (refs.size() > 0) {
+            for (Iterator<Reference> j = refs.iterator(); j.hasNext();) {
+                Reference r = (Reference) j.next();
+
+                long bytesTransferred = getBytesTransferred(r);
+
+                if (!isDir(r)) {
+                    // only add this if > 0
+                    if (bytesTransferred > 0) {
+                        totalBytesTransferred += bytesTransferred;
+                    }
+
+                    // add this no matter what
+                    totalProductSize += r.getFileSize();
+                }
+
+            }
+        }
+
+        return ((double) ((1.0 * totalBytesTransferred) / (1.0 * totalProductSize)));
+
+    }
+
+    public double getPctTransferred(Reference ref) {
+        long bytesTransferred = getBytesTransferred(ref);
+        return ((double) ((1.0 * bytesTransferred) / (1.0 * ref.getFileSize())));
+    }
+
+    public void removeProductTransferStatus(Product product) {
+        if (currentProductTransfers.get(product.getProductId()) != null) {
+            currentProductTransfers.remove(product.getProductId());
+        }
+    }
+
+    public boolean isTransferComplete(Product product) {
+        return getPctTransferred(product) == 1.0;
+    }
+
+    private long getBytesTransferred(Reference r) {
+        File destFile = null;
+
+        try {
+            destFile = new File(new URI(r.getDataStoreReference()));
+            long bytesTransferred = destFile.length();
+            return bytesTransferred;
+        } catch (URISyntaxException e) {
+            LOG.log(Level.WARNING,
+                    "URISyntaxException when checking size of destFile: ["
+                            + r.getDataStoreReference() + "]: Message: "
+                            + e.getMessage());
+            return -1L;
+        }
+    }
+
+    private List<Reference> quietGetReferences(Product p) {
+        List<Reference> refs = null;
+
+        try {
+            refs = catalog.getProductReferences(p);
+        } catch (CatalogException e) {
+            e.printStackTrace();
+            LOG.log(Level.WARNING, "Error retreiving references for product: ["
+                    + p.getProductId()
+                    + "] from catalog in transfer status tracker: Message: "
+                    + e.getMessage());
+            refs = new Vector<Reference>();
+        }
+
+        return refs;
+    }
+
+    private boolean isDir(Reference r) {
+        File fileRef = null;
+
+        try {
+            fileRef = new File(new URI(r.getDataStoreReference()));
+            return fileRef.isDirectory();
+        } catch (URISyntaxException e) {
+            return false;
+        }
+    }
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/AbstractCacheServerFactory.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/AbstractCacheServerFactory.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/AbstractCacheServerFactory.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/AbstractCacheServerFactory.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//OODT imports
+import org.apache.oodt.cas.metadata.util.PathUtils;
+
+//JDK imports
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * Describe your class here
+ * </p>.
+ */
+public abstract class AbstractCacheServerFactory implements CacheFactory {
+
+    protected String rangeQueryElementName;
+
+    protected String rangeStartDateTime;
+
+    protected String rangeEndDateTime;
+
+    protected String uniqueElementName;
+
+    protected List<String> productTypeNames;
+
+    protected URL fmUrl;
+
+    public AbstractCacheServerFactory() throws InstantiationException {
+        rangeQueryElementName = System
+                .getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest.cache.rangeQueryElementName");
+
+        // before we replace env vars, try to replace date vars
+        rangeStartDateTime = System
+                .getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest.cache.range.start");
+        rangeEndDateTime = System
+                .getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest.cache.range.end");
+
+        try {
+            rangeStartDateTime = PathUtils
+                    .doDynamicReplacement(rangeStartDateTime);
+            rangeEndDateTime = PathUtils.doDynamicReplacement(rangeEndDateTime);
+        } catch (Exception e) {
+            throw new InstantiationException(e.getMessage());
+        }
+
+        uniqueElementName = System
+                .getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest.cache.uniqueElementName");
+        productTypeNames = Arrays.asList(PathUtils.replaceEnvVariables(
+                System.getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest."
+                        + "cache.productType")).split(","));
+
+        try {
+            fmUrl = new URL(
+                    System
+                            .getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest.cache.filemgr.url"));
+        } catch (MalformedURLException e) {
+            throw new InstantiationException(
+                    "Unable to construct file manager url for: ["
+                            + System
+                                    .getProperty("gov.nasa.jpl.oodt.cas.filemgr.ingest.cache.filemgr.url")
+                            + "]: malformed URL exception.");
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.CacheFactory#createCache()
+     */
+    public abstract Cache createCache() throws InstantiationException;
+
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Cache.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Cache.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Cache.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Cache.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//JDK imports
+import java.net.URL;
+import java.util.List;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.exceptions.CacheException;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * The core interface for a cache of {@link Product}s (identified by a
+ * particular <code>uniqueElement</code>) from a File Manager.
+ * </p>.
+ */
+public interface Cache {
+
+    public static final String DEFAULT_UNIQUE_MET_KEY = "CAS.ProductName";
+
+    /**
+     * Clears the current cache. This should be used before a re-{@link #sync(String, String))}
+     * occurs.
+     * 
+     */
+    public void clear();
+
+    /**
+     * Checks the cache to determine if a {@link Product} with the given
+     * <code>productName</code> exists.
+     * 
+     * @param productName
+     *            The name of the {@link Product} to check for in the Cache.
+     * @return True if the {@link Product} is foudn in the Cache, False,
+     *         otherwise.
+     */
+    public boolean contains(String productName);
+
+    /**
+     * Specifies the {@link URL} to the File Manager to connect this Cache to.
+     * 
+     * @param fmUrl
+     *            The {@link URL} of the File Manager to cache {@link Product}s
+     *            from.
+     */
+    public void setFileManager(URL fmUrl);
+
+    /**
+     * 
+     * @return The size of the current {@link Product} cache.
+     */
+    public int size();
+
+    /**
+     * Synchronizes this Cache with the File Manager specified by the
+     * {@link #setFileManager(URL)} method.
+     * 
+     * @param uniqueElementProductTypeNames
+     *            The {@link List} {@link ProductType}s to cache
+     *            {@link Product}s from.
+     * @throws CacheException
+     *             If there is any error.
+     */
+    public void sync(List<String> uniqueElementProductTypeNames) throws CacheException;
+
+    /**
+     * Synchronizes this Cache with the File Manager specified by the
+     * {@link #setFileManager(URL)} method.
+     * 
+     * @param uniqueElementName
+     *            The met key identifier to use for determining {@link Product}
+     *            existance (if different than {@value #DEFAULT_UNIQUE_MET_KEY}).
+     * @param uniqueElementProductTypeNames
+     *            The {@link List} of {@link ProductType}s to cache
+     *            {@link Product}s from.
+     * @throws CacheException
+     *             If there is any error.
+     */
+    public void sync(String uniqueElementName,
+            List<String> uniqueElementProductTypeNames) throws CacheException;
+
+    /**
+     * Synchronizes this Cache with the File Manager specified by the
+     * {@link #setFileManager(URL)} method. This method also assumes that the
+     * unique element (identified by {@link #setUniqueElementName(String)} and
+     * the {@link List} of {@link ProductType}s have also been set appropriate
+     * (e.g., by using {@link #setUniqueElementProductTypeNames(List)}.
+     * 
+     * @throws CacheException
+     *             If any error occurs.
+     */
+    public void sync() throws CacheException;
+
+    /**
+     * Sets the names of the {@link ProductType}s to cache.
+     * 
+     * @param uniqueElementProductTypeNames
+     *            A {@link List} of java.util.String names of
+     *            {@link ProductType}s.
+     */
+    public void setUniqueElementProductTypeNames(
+            List<String> uniqueElementProductTypeNames);
+
+    /**
+     * Sets the name of the met element to use as the identifier of a
+     * {@link Product} for use in determining whether the Product is in the
+     * Cache.
+     * 
+     * @param uniqueElementName
+     *            The name of the met element used to uniquely identify
+     *            {@link Product}s.
+     */
+    public void setUniqueElementName(String uniqueElementName);
+
+    /**
+     * Gets the {@link URL} of the File Manager that this Cache communicates
+     * with.
+     * 
+     * @return The {@link URL} of the File Manager that this Cache communicates
+     *         with.
+     */
+    public URL getFileManagerUrl();
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CacheFactory.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CacheFactory.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CacheFactory.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CacheFactory.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+/**
+ * @author mattmann
+ * @version $Revision$
+ * 
+ * <p>
+ * A Factory class for creating {@link Cache}s
+ * </p>.
+ */
+public interface CacheFactory {
+
+    /**
+     * Create a new {@link Cache}.
+     * 
+     * @return A new {@link Cache}, constructed by the implementing Factory.
+     * @throws InstantiationException If the the particular {@link Cache} can
+     * not be created for some reason.
+     */
+    public Cache createCache() throws InstantiationException;
+
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//JDK imports
+import java.io.File;
+import java.io.FileInputStream;
+import java.net.URL;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.exceptions.CacheException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
+
+/**
+ * @author mattmann
+ * @version $Revision$
+ * 
+ * <p>
+ * An extension of the {@link StdIngester} that uses a {@link Cache} to keep
+ * track of {@link Product} ingestion status. If the existing {@link Cache} used
+ * is already sync'ed to the requested File Manager (specified by the
+ * <code>fmUrl</code> parameter in {@link #hasProduct(URL, File)} or
+ * {@link #hasProduct(URL, String)}), then the {@link Cache} will simply return
+ * the value of {@link Cache#contains(String)}. Otherwise the {@link Cache}
+ * will be re-{@link Cache#sync()}'ed to the given File Manager before the
+ * contains method is invoked.
+ * </p>.
+ */
+public class CachedIngester extends StdIngester {
+
+    private Cache cache;
+
+    private static final Logger LOG = Logger.getLogger(CachedIngester.class
+            .getName());
+
+    /**
+     * @param transferService
+     *            The underlying data transfer service to use to ingest
+     *            {@link Product}s.
+     * @param cacheServiceFactory
+     *            The {@link CacheFactory} to use to construct this
+     *            {@link Ingester}'s {@link Cache}.
+     * @param cachePropFile
+     *            The file path to the cache properties file to load to
+     *            configure the {@link Cache}.
+     */
+    public CachedIngester(String transferService, String cacheServiceFactory,
+            String cachePropFile) throws InstantiationException {
+        super(transferService);
+        try {
+            System.getProperties().load(new FileInputStream(cachePropFile));
+        } catch (Exception e) {
+            throw new InstantiationException(
+                    "Unable to load cache properties from file: ["
+                            + cachePropFile + "]");
+        }
+        this.cache = GenericFileManagerObjectFactory
+                .getCacheFromFactory(cacheServiceFactory);
+        init(this.cache);
+
+    }
+
+    /**
+     * 
+     * @param transferService
+     *            The underlying data transfer service to use to ingest
+     *            {@link Product}s.
+     * @param cache
+     *            The {@link Cache} that this {@link Ingester} will use.
+     * @throws InstantiationException
+     *             If any error occurs.
+     */
+    public CachedIngester(String transferService, Cache cache)
+            throws InstantiationException {
+        super(transferService);
+        init(cache);
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.StdIngester#hasProduct(java.net.URL,
+     *      java.io.File)
+     */
+    public boolean hasProduct(URL fmUrl, File prodFile) throws CatalogException {
+        return hasProduct(fmUrl, prodFile.getName());
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.StdIngester#hasProduct(java.net.URL,
+     *      java.lang.String)
+     */
+    public boolean hasProduct(URL fmUrl, String productName)
+            throws CatalogException {
+        if (cache.getFileManagerUrl().equals(fmUrl)) {
+            return cache.contains(productName);
+        } else {
+            // need to re-sync
+            cache.setFileManager(fmUrl);
+            try {
+                cache.sync();
+            } catch (CacheException e) {
+                LOG.log(Level.WARNING,
+                        "Exception re-syncing cache to file manager: [" + fmUrl
+                                + "]: Message: " + e.getMessage());
+                throw new CatalogException(
+                        "Exception re-syncing cache to file manager: [" + fmUrl
+                                + "]: Message: " + e.getMessage());
+            }
+            return cache.contains(productName);
+        }
+    }
+
+    /**
+     * 
+     * @throws CacheException
+     */
+    public void resynsc() throws CacheException {
+        cache.sync();
+    }
+
+    private void init(Cache cache) throws InstantiationException {
+        this.cache = cache;
+        // upon inception, do an initial sync
+        try {
+            cache.sync();
+        } catch (CacheException e) {
+            throw new InstantiationException(
+                    "Unable to sync cache for CachedIngester: Message: "
+                            + e.getMessage());
+        }
+    }
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.exceptions.IngestException;
+import org.apache.oodt.cas.metadata.MetExtractor;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+import org.apache.oodt.cas.metadata.util.GenericMetadataObjectFactory;
+
+//JDK imports
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * A command line I/F for {@link Product} ingestion.
+ * </p>.
+ */
+public class CmdLineIngester extends StdIngester {
+
+    /* our log stream */
+    private static final Logger LOG = Logger.getLogger(CmdLineIngester.class
+            .getName());
+
+    public CmdLineIngester(String serviceFactory) {
+        super(serviceFactory);
+    }
+
+    /**
+     * @param args
+     * @throws IngestException
+     * @throws MalformedURLException
+     */
+    public static void main(String[] args) throws Exception {
+        String usage = CmdLineIngester.class.getName()
+                + " --url <filemgr url> [options]\n"
+                + "[--extractor <met extractor class name> <met conf file path>|"
+                + "--metFile <met file path>]\n"
+                + "--transferer <data transfer service factory>\n"
+                + "[--file <full path> | --in reads list of files from STDIN]\n";
+
+        String fmUrlStr = null, extractorClassName = null, filePath = null;
+        String transferServiceFactory = null, metConfFilePath = null;
+        String metFilePath = null;
+        boolean readFromStdin = false;
+
+        for (int i = 0; i < args.length; i++) {
+            if (args[i].equals("--url")) {
+                fmUrlStr = args[++i];
+            } else if (args[i].equals("--extractor")) {
+                extractorClassName = args[++i];
+                metConfFilePath = args[++i];
+            } else if (args[i].equals("--file")) {
+                filePath = args[++i];
+            } else if (args[i].equals("--in")) {
+                readFromStdin = true;
+            } else if (args[i].equals("--transferer")) {
+                transferServiceFactory = args[++i];
+            } else if (args[i].equals("--metFile")) {
+                metFilePath = args[++i];
+            }
+        }
+
+        if (fmUrlStr == null
+                || ((extractorClassName == null || metConfFilePath == null) && metFilePath == null)
+                || (metFilePath != null && (metConfFilePath != null || extractorClassName != null))
+                || transferServiceFactory == null
+                || (filePath == null && !readFromStdin)
+                || (readFromStdin && metFilePath != null)
+                || (readFromStdin && filePath != null)) {
+            System.err.println(usage);
+            System.exit(1);
+        }
+
+        CmdLineIngester ingester = new CmdLineIngester(transferServiceFactory);
+        MetExtractor extractor = null;
+        if (readFromStdin) {
+            List prods = readProdFilesFromStdin();
+            extractor = GenericMetadataObjectFactory
+                    .getMetExtractorFromClassName(extractorClassName);
+            ingester.ingest(new URL(fmUrlStr), prods, extractor, new File(
+                    metConfFilePath));
+        } else {
+            String productID = null;
+            if (metFilePath != null) {
+                productID = ingester.ingest(new URL(fmUrlStr), new File(
+                        filePath), new SerializableMetadata(new FileInputStream(metFilePath)));
+            } else {
+                extractor = GenericMetadataObjectFactory
+                        .getMetExtractorFromClassName(extractorClassName);
+                productID = ingester.ingest(new URL(fmUrlStr), new File(
+                        filePath), extractor, new File(metConfFilePath));
+            }
+
+            System.out.println("Result: " + productID);
+        }
+
+    }
+
+    private static List<String> readProdFilesFromStdin() {
+        List<String> prodFiles = new Vector<String>();
+        BufferedReader br = null;
+
+        br = new BufferedReader(new InputStreamReader(System.in));
+
+        String line = null;
+
+        try {
+            while ((line = br.readLine()) != null) {
+                prodFiles.add(line);
+            }
+        } catch (IOException e) {
+            LOG.log(Level.WARNING, "Error reading prod file: line: [" + line
+                    + "]: Message: " + e.getMessage(), e);
+        }
+
+        return prodFiles;
+    }
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.IngestException;
+import org.apache.oodt.cas.metadata.MetExtractor;
+import org.apache.oodt.cas.metadata.Metadata;
+
+//JDK imports
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ * An interface for ingesting {@link Product}s
+ * </p>.
+ */
+public interface Ingester {
+
+    /**
+     * Ingests a {@link Product} to the file manager service object identified
+     * by the given {@link URL} parameter. The product {@link Metadata} is
+     * extracted dynamically using the provided {@link MetExtractor} interface.
+     * 
+     * @param fmUrl
+     *            The {@link URL} pointer to the file manager service.
+     * @param prodFile
+     *            The {@link File} pointer to the product file.
+     * @param extractor
+     *            The given {@link MetExtractor} to use to extract
+     *            {@link Metadata} from the {@link Product}.
+     * @param metConfFile
+     *            A Config{@link File} for the {@link MetExtractor}.
+     * @return The ID returned by the file manager for the newly ingested
+     *         product.
+     * @throws IngestException
+     *             If there is an error ingesting the {@link Product}
+     */
+    public String ingest(URL fmUrl, File prodFile, MetExtractor extractor,
+            File metConfFile) throws IngestException;
+
+    /**
+     * Ingests a {@link Product} to the file manager service object identified
+     * by the given {@link URL} parameter. The product {@link Metadata} is
+     * provided a priori.
+     * 
+     * @param fmUrl
+     *            The {@link URL} pointer to the file manager service.
+     * @param prodFile
+     *            The {@link File} pointer to the product file.
+     * @param met
+     *            The given {@link Metadata} object already extracted from the
+     *            {@link Product}.
+     * @return The ID returned by the file manager for the newly ingested
+     *         product.
+     * @throws IngestException
+     *             If there is an error ingesting the {@link Product}
+     */
+    public String ingest(URL fmUrl, File prodFile, Metadata met)
+            throws IngestException;
+
+    /**
+     * 
+     * @param fmUrl
+     *            The {@link URL} pointer to the file manager service.
+     * @param prodFiles
+     *            A {@link List} of {@link String} filePaths pointing to
+     *            {@link Product} files to ingest.
+     * @param extractor
+     *            The given {@link MetExtractor} to use to extract
+     *            {@link Metadata} from the {@link Product}s.
+     * @param metConfFile
+     *            A Config{@link File} for the {@link MetExtractor}.
+     * @throws IngestException
+     *             If there is an error ingesting the {@link Product}s.
+     */
+    public void ingest(URL fmUrl, List<String> prodFiles, MetExtractor extractor,
+            File metConfFile) throws IngestException;
+
+    /**
+     * Checks the file manager at the given {@link URL} to see whether or not it
+     * knows about the provided {@link Product} {@link File} parameter. To do
+     * this, it uses {@link File#getName()} as the {@link Metadata} key
+     * <code>Filename</code>.
+     * 
+     * @param prodFile
+     *            The {@link File} to check for existance of within the file
+     *            manager at given {@link URL}.
+     * @url The {@link URL} pointer to the file manager service.
+     * @return
+     */
+    public boolean hasProduct(URL fmUrl, File prodFile) throws CatalogException;
+
+    /**
+     * Checks the file manager at the given {@link URL} to see whether or not it
+     * knows about the provided {@link Product} with the given
+     * <code>productName</code> parameter. To do this, it uses the provided
+     * <code>productName</code> key as the {@link Metadata} key to search for
+     * in the catalog.
+     * 
+     * @param fmUrl
+     *            The {@link URL} pointer to the file manager service.
+     * @param productName
+     *            The {@link Product} to search for, identified by its (possibly
+     *            not unique) name.
+     * @return True if the file manager has the product, false otherwise.
+     */
+    public boolean hasProduct(URL fmUrl, String productName) throws CatalogException;
+
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//JDK imports
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Query;
+import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CacheException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+
+/**
+ * 
+ * @author bfoster
+ * @author mattmann
+ * 
+ * <p>
+ * A cached way to determine product ingestion against a File Manager.
+ * </p>
+ * 
+ */
+public class LocalCache implements Cache {
+
+    private HashSet<String> uniqueElements;
+
+    private String uniqueElementName;
+
+    private List<String> uniqueElementProductTypeNames;
+
+    private String rangeQueryElementName;
+
+    private String startOfQuery;
+
+    private String endOfQuery;
+
+    private XmlRpcFileManagerClient fm;
+
+    private static final Logger LOG = Logger.getLogger(LocalCache.class
+            .getName());
+
+    /**
+     * 
+     * @param fmUrl
+     * @param uniqueElementName
+     * @param rangeQueryElementName
+     * @param startOfQuery
+     * @param endOfQuery
+     */
+    public LocalCache(URL fmUrl, String rangeQueryElementName,
+            String startOfQuery, String endOfQuery) {
+        this(fmUrl, null, null, rangeQueryElementName, startOfQuery, endOfQuery);
+
+    }
+
+    /**
+     * 
+     * @param fmUrl
+     * @param uniqueElementName
+     * @param uniqueElementProductType
+     * @param rangeQueryElementName
+     * @param startOfQuery
+     * @param endOfQuery
+     */
+    public LocalCache(URL fmUrl, String uniqueElementName,
+            List<String> uniqueElementProductTypes, String rangeQueryElementName,
+            String startOfQuery, String endOfQuery) {
+        this.uniqueElements = new HashSet<String>();
+        this.uniqueElementName = uniqueElementName;
+        this.uniqueElementProductTypeNames = uniqueElementProductTypes;
+        this.rangeQueryElementName = rangeQueryElementName;
+        this.startOfQuery = startOfQuery;
+        this.endOfQuery = endOfQuery;
+        setFileManager(fmUrl);
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#sync(java.util.List)
+     */
+    public void sync(List<String> uniqueElementProductTypeNames) throws CacheException {
+        sync(DEFAULT_UNIQUE_MET_KEY, uniqueElementProductTypeNames);
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#sync(java.lang.String,
+     *      java.util.List)
+     */
+    public void sync(String uniqueElementName,
+            List<String> uniqueElementProductTypeNames) throws CacheException {
+        try {
+            this.uniqueElementName = uniqueElementName;            
+            this.uniqueElementProductTypeNames = uniqueElementProductTypeNames;
+            List<Product> products = new Vector<Product>();
+            for (String productType : this.uniqueElementProductTypeNames)
+                products.addAll(getProductsOverDateRange(
+                    this.rangeQueryElementName, productType, this.startOfQuery,
+                    this.endOfQuery));
+            clear();
+            for (Iterator<Product> i = products.iterator(); i.hasNext();) {
+                Product product = i.next();
+
+                String value = getValueForMetadata(product, uniqueElementName);
+                this.uniqueElements.add(value);
+            }
+        } catch (Exception e) {
+            throw new CacheException("Failed to sync with database : "
+                    + e.getMessage());
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#clear()
+     */
+    public void clear() {
+        this.uniqueElements.clear();
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#size()
+     */
+    public int size() {
+        return this.uniqueElements.size();
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#contains(java.lang.String)
+     */
+    public boolean contains(String productName) {
+        return this.uniqueElements.contains(productName);
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#setFileManager(java.net.URL)
+     */
+    public void setFileManager(URL fmUrl) {
+        try {
+            this.fm = new XmlRpcFileManagerClient(fmUrl);
+        } catch (ConnectionException e) {
+            LOG.log(Level.WARNING,
+                    "Exception setting file manager connection to: [" + fmUrl
+                            + "]");
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#sync()
+     */
+    public void sync() throws CacheException {
+        if (this.uniqueElementName == null
+                || (this.uniqueElementProductTypeNames == null || 
+                        (this.uniqueElementProductTypeNames != null && this.uniqueElementProductTypeNames.size() == 0))) {
+            throw new CacheException(
+                    "Both uniqueElementName and uniqueElementProductTypeName must "
+                            + "be defined in order to use this form of the sync operation!");
+        }
+
+        sync(this.uniqueElementName, this.uniqueElementProductTypeNames);
+
+    }
+    
+    /* (non-Javadoc)
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.Cache#getFileManagerUrl()
+     */
+    public URL getFileManagerUrl() {
+        return this.fm.getFileManagerUrl();
+    }
+
+    /**
+     * Performs a {@link XmlRpcFileManagerClient#hasProduct(String)} check
+     * against a live File Manager, bypassing the {@link Cache}.
+     * 
+     * @param uniqueElementName
+     *            The product identifier element to identify whether the product
+     *            was received yet.
+     * @param uniqueElementValue
+     *            The value of the product identifier element.
+     * @param productTypeName
+     *            The {@link ProductType} of the Product that you would like to
+     *            check existence for.
+     * @return True if the given Product (identified by its
+     *         <code>uniqueElement</code>) exists, False otherwise.
+     * @throws CacheException
+     *             If any error occurs.
+     */
+    public boolean liveHasProduct(String uniqueElementName,
+            String uniqueElementValue, String productTypeName)
+            throws CacheException {
+        Query query = new Query();
+        query.addCriterion(new TermQueryCriteria(uniqueElementName,
+                uniqueElementValue));
+        try {
+            return !(fm.query(query, fm.getProductTypeByName(productTypeName))
+                    .isEmpty());
+        } catch (Exception e) {
+            throw new CacheException(
+                    "Unable to check for product reception from file manager: ["
+                            + fm.getFileManagerUrl() + "]: Message: "
+                            + e.getMessage());
+        }
+    }
+
+    /**
+     * @return the uniqueElementName
+     */
+    public String getUniqueElementName() {
+        return uniqueElementName;
+    }
+
+    /**
+     * @param uniqueElementName
+     *            the uniqueElementName to set
+     */
+    public void setUniqueElementName(String uniqueElementName) {
+        this.uniqueElementName = uniqueElementName;
+    }
+
+    /**
+     * @return the uniqueElementProductTypeNames
+     */
+    public List<String> getUniqueElementProductTypeNames() {
+        return uniqueElementProductTypeNames;
+    }
+
+    /**
+     * @param uniqueElementProductTypeNames
+     *            the uniqueElementProductTypeNames to set
+     */
+    public void setUniqueElementProductTypeNames(
+            List<String> uniqueElementProductTypeNames) {
+        this.uniqueElementProductTypeNames = uniqueElementProductTypeNames;
+    }
+
+    private List<Product> getProductsOverDateRange(String elementName, String productType,
+            String startOfQuery, String endOfQuery) throws CacheException {
+        List<Product> products = new Vector<Product>();
+        try {        
+            Query query = new Query();
+            query.addCriterion(new RangeQueryCriteria(elementName,
+                    startOfQuery, endOfQuery));
+            if(this.uniqueElementProductTypeNames != null && 
+                    this.uniqueElementProductTypeNames.size() > 0){
+                for(Iterator<String> i = this.uniqueElementProductTypeNames.iterator(); i.hasNext(); ){
+                    String productTypeName = i.next();
+                    products.addAll(getProducts(query, productTypeName));
+                }
+                
+            }
+        } catch (Exception e) {
+            throw new CacheException("Failed to query for product via element "
+                    + elementName + " and range " + startOfQuery + " to "
+                    + endOfQuery + " : " + e.getMessage());
+        }
+        
+        return products;
+    }
+
+    private List<Product> getProducts(Query query, String productType)
+            throws CacheException {
+        try {
+            return fm.query(query, fm.getProductTypeByName(productType));
+        } catch (Exception e) {
+            throw new CacheException("Failed to get product list for query "
+                    + query + " : " + e.getMessage());
+        }
+    }
+
+    private String getValueForMetadata(Product product,
+            String metadataElementName) throws CacheException {
+        try {
+            return fm.getMetadata(product).getMetadata(metadataElementName);
+        } catch (Exception e) {
+            throw new CacheException("Failed to get metadata value for "
+                    + metadataElementName + " : " + e.getMessage());
+        }
+    }
+
+}
\ No newline at end of file

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCacheFactory.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCacheFactory.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCacheFactory.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCacheFactory.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+/**
+ * @author mattmann
+ * @version $Revision$
+ * 
+ * <p>
+ * Constructs new {@link LocalCache}s
+ * </p>.
+ */
+public class LocalCacheFactory extends AbstractCacheServerFactory {
+
+    public LocalCacheFactory() throws InstantiationException {
+        super();
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see gov.nasa.jpl.oodt.cas.filemgr.ingest.AbstractCacheServerFactory#createCache()
+     */
+    public Cache createCache() throws InstantiationException {
+        return new LocalCache(fmUrl, uniqueElementName, productTypeNames,
+                rangeQueryElementName, rangeStartDateTime, rangeEndDateTime);
+    }
+
+}

Added: oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/RemoteableCache.java
URL: http://svn.apache.org/viewvc/oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/RemoteableCache.java?rev=1052148&view=auto
==============================================================================
--- oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/RemoteableCache.java (added)
+++ oodt/branches/wengine-branch/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/RemoteableCache.java Thu Dec 23 02:48:02 2010
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.ingest;
+
+//JDK imports
+import java.net.URL;
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+import java.util.List;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ * 
+ * <p>
+ *  A Java RMI {@link Remote}-able implementation of the {@link Cache}
+ *  interface.
+ * </p>.
+ */
+public interface RemoteableCache extends Remote{
+
+    public static final String DEFAULT_UNIQUE_MET_KEY = "CAS.ProductName";
+
+    /**
+     * Clears the current cache. This should be used before a re-{@link #sync(String, String))}
+     * occurs.
+     * 
+     */
+    public void clear() throws RemoteException;
+
+    /**
+     * Checks the cache to determine if a {@link Product} with the given
+     * <code>productName</code> exists.
+     * 
+     * @param productName
+     *            The name of the {@link Product} to check for in the Cache.
+     * @return True if the {@link Product} is foudn in the Cache, False,
+     *         otherwise.
+     */
+    public boolean contains(String productName) throws RemoteException;
+
+    /**
+     * Specifies the {@link URL} to the File Manager to connect this Cache to.
+     * 
+     * @param fmUrl
+     *            The {@link URL} of the File Manager to cache {@link Product}s
+     *            from.
+     */
+    public void setFileManager(URL fmUrl) throws RemoteException;
+
+    /**
+     * 
+     * @return The size of the current {@link Product} cache.
+     */
+    public int size() throws RemoteException;
+
+    /**
+     * Synchronizes this Cache with the File Manager specified by the
+     * {@link #setFileManager(URL)} method.
+     * 
+     * @param uniqueElementProductTypeNames
+     *            The {@link List} {@link ProductType}s to cache
+     *            {@link Product}s from.
+     * @throws RemoteException
+     *             If there is any error.
+     */
+    public void sync(List<String> uniqueElementProductTypeNames) throws RemoteException;
+
+    /**
+     * Synchronizes this Cache with the File Manager specified by the
+     * {@link #setFileManager(URL)} method.
+     * 
+     * @param uniqueElementName
+     *            The met key identifier to use for determining {@link Product}
+     *            existance (if different than {@value #DEFAULT_UNIQUE_MET_KEY}).
+     * @param uniqueElementProductTypeNames
+     *            The {@link List} of {@link ProductType}s to cache
+     *            {@link Product}s from.
+     * @throws RemoteException
+     *             If there is any error.
+     */
+    public void sync(String uniqueElementName,
+            List<String> uniqueElementProductTypeNames) throws RemoteException;
+
+    /**
+     * Synchronizes this Cache with the File Manager specified by the
+     * {@link #setFileManager(URL)} method. This method also assumes that the
+     * unique element (identified by {@link #setUniqueElementName(String)} and
+     * the {@link List} of {@link ProductType}s have also been set appropriate
+     * (e.g., by using {@link #setUniqueElementProductTypeNames(List)}.
+     * 
+     * @throws RemoteException
+     *             If any error occurs.
+     */
+    public void sync() throws RemoteException;
+
+    /**
+     * Sets the names of the {@link ProductType}s to cache.
+     * 
+     * @param uniqueElementProductTypeNames
+     *            A {@link List} of java.util.String names of
+     *            {@link ProductType}s.
+     */
+    public void setUniqueElementProductTypeNames(
+            List<String> uniqueElementProductTypeNames) throws RemoteException;
+
+    /**
+     * Sets the name of the met element to use as the identifier of a
+     * {@link Product} for use in determining whether the Product is in the
+     * Cache.
+     * 
+     * @param uniqueElementName
+     *            The name of the met element used to uniquely identify
+     *            {@link Product}s.
+     */
+    public void setUniqueElementName(String uniqueElementName)
+            throws RemoteException;
+
+    /**
+     * Gets the {@link URL} of the File Manager that this Cache communicates
+     * with.
+     * 
+     * @return The {@link URL} of the File Manager that this Cache communicates
+     *         with.
+     */
+    public URL getFileManagerUrl() throws RemoteException;
+}