oodt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mattm...@apache.org
Subject [08/88] [abbrv] [partial] oodt git commit: Record change for OODT-836, OODT-837.
Date Mon, 17 Jul 2017 23:05:23 GMT
http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/DirStructXmlParser.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/DirStructXmlParser.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/DirStructXmlParser.java
deleted file mode 100644
index d89936e..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/DirStructXmlParser.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.filerestrictions.parsers;
-
-import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.metadata.exceptions.CasMetadataException;
-import org.apache.oodt.cas.metadata.util.PathUtils;
-import org.apache.oodt.cas.pushpull.exceptions.ParserException;
-import org.apache.oodt.cas.pushpull.expressions.GlobalVariables;
-import org.apache.oodt.cas.pushpull.expressions.Method;
-import org.apache.oodt.cas.pushpull.expressions.Variable;
-import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFile;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFileStructure;
-import org.apache.oodt.commons.exceptions.CommonsException;
-import org.apache.oodt.commons.xml.XMLUtils;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-
-import java.io.FileInputStream;
-import java.text.ParseException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.StringTokenizer;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.xml.parsers.DocumentBuilderFactory;
-
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- * 
- * <p>
- * Describe your class here
- * </p>.
- */
-public class DirStructXmlParser implements Parser {
-
-    private static final Logger LOG = Logger.getLogger(DirStructXmlParser.class
-            .getName());
-    
-    private static final ConcurrentHashMap<String, Method> methodRepo = new ConcurrentHashMap<String, Method>();
-
-    public DirStructXmlParser() {}
-
-    public VirtualFileStructure parse(FileInputStream xmlFile, Metadata metadata)
-            throws ParserException {
-        try {
-            String initialCdDir = "/";
-            VirtualFile root = null;
-            NodeList list = (DocumentBuilderFactory.newInstance()
-                    .newDocumentBuilder().parse(new InputSource(xmlFile)))
-                    .getDocumentElement().getChildNodes();
-            VirtualFile currentFile;
-            for (int i = 0; i < list.getLength(); i++) {
-                Node node = list.item(i);
-                if (node.getNodeName().equals("dirstruct")) {
-
-                    // parse out starting path
-                    String startingPath = ((Element) node)
-                            .getAttribute("starting_path");
-                        root = (currentFile = new VirtualFile(
-                                initialCdDir = startingPath, true))
-                                .getRootDir();
-                        VirtualFile temp = currentFile.getParentFile();
-                        while (temp != null) {
-                            temp.setNoDirs(true);
-                            temp.setNoFiles(true);
-                            temp = temp.getParentFile();
-                        }
-                    // parse the directory structure
-                    parseDirstructXML(node.getChildNodes(), currentFile);
-
-                } else if (node.getNodeName().equals("variables")) {
-                    parseVariablesXML(node.getChildNodes());
-                } else if (node.getNodeName().equals("methods")) {
-                    parseMethodsXML(node.getChildNodes());
-                }
-            }
-            return new VirtualFileStructure(initialCdDir, root);
-        } catch (Exception e) {
-            LOG.log(Level.SEVERE, e.getMessage());
-            throw new ParserException("Failed to parse XML file : "
-                    + e.getMessage());
-        }
-    }
-
-    private void parseDirstructXML(NodeList list, VirtualFile currentLoadFile) {
-        for (int i = 0; i < list.getLength(); i++) {
-            Node dir = list.item(i);
-            if (dir.getNodeName().equals("dir")) {
-                String dirName = replaceVariablesAndMethods(((Element) dir)
-                        .getAttribute("name"));
-                currentLoadFile.addChild(new VirtualFile(dirName, true));
-                NodeList children;
-                if ((children = dir.getChildNodes()).getLength() > 0) {
-                    parseDirstructXML(children, currentLoadFile.getChild(
-                            dirName, true));
-                }
-            } else if (dir.getNodeName().equals("nodirs")) {
-                currentLoadFile.setNoDirs(true);
-            } else if (dir.getNodeName().equals("nofiles")) {
-                currentLoadFile.setNoFiles(true);
-            } else if (dir.getNodeName().equals("file")) {
-                VirtualFile vf = new VirtualFile(
-                        replaceVariablesAndMethods(((Element) dir)
-                                .getAttribute("name")), false);
-                vf.setNoDirs(true);
-                currentLoadFile.addChild(vf);
-            }
-        }
-    }
-
-    private String replaceVariablesAndMethods(String input) {
-        for (int i = 0; i < input.length(); i++) {
-            char c = input.charAt(i);
-            switch (c) {
-            case '$':
-                try {
-                    if (input.charAt(i + 1) == '{') {
-                        StringBuilder variable = new StringBuilder("");
-                        for (int j = i + 2; j < input.length(); j++) {
-                            char ch = input.charAt(j);
-                            if ((ch <= 'Z' && ch >= 'A')
-                                    || (ch <= 'z' && ch >= 'a')
-                                    || (ch <= '9' && ch >= '0') 
-                                    || ch == '_') {
-                              variable.append(ch);
-                            } else {
-                              break;
-                            }
-                        }
-                        Variable v = GlobalVariables.ConcurrentHashMap.get(variable
-                                .toString());
-                        if (v == null) {
-                          throw new Exception("No variable defined with name '" + variable.toString() + "'");
-                        }
-                        input = input.replaceFirst("\\$\\{" + variable + "\\}", v.toString());
-                        i = i + v.toString().length();
-                    }
-                } catch (Exception e) {
-                	LOG.log(Level.WARNING, "Failed to replace variable in '" + input + " for i = '" + i + "' : " + e.getMessage(), e);
-                }
-                break;
-            case '%':
-                try {
-                    StringBuilder method = new StringBuilder("");
-                    int j = i + 1;
-                    for (; j < input.length(); j++) {
-                        char ch = input.substring(j, j + 1).charAt(0);
-                        if ((ch <= 'Z' && ch >= 'A')
-                                || (ch <= 'z' && ch >= 'a')
-                                || (ch <= '9' && ch >= '0') || ch == '_') {
-                          method.append(ch);
-                        } else {
-                          break;
-                        }
-                    }
-
-                    if (input.substring(j, j + 1).charAt(0) == '(') {
-                        Method m = methodRepo.get(method.toString());
-                        StringTokenizer st = new StringTokenizer(input
-                                .substring(j, input.substring(j).indexOf(")")
-                                        + j), "#\", ()");
-                        while (st.hasMoreTokens()) {
-                            String arg = st.nextToken();
-                            m.addArg(null, arg);
-                        }
-                        String returnValue = m.execute().toString();
-                        input = input.substring(0, i)
-                                + returnValue
-                                + input.substring(input.substring(i).indexOf(
-                                        ")")
-                                        + 1 + i);
-                        i = i + returnValue.length();
-                    } else {
-                        LOG.log(Level.SEVERE, "Invalid method signature in "
-                                + input + " near " + method);
-                        break;
-                    }
-                } catch (Exception ignored) {
-                }
-                break;
-            }
-        }
-        return input;
-    }
-
-    private void parseVariablesXML(NodeList list) throws ParseException, CommonsException, CasMetadataException {
-
-        // loop through all variable elements
-        for (int i = 0; i < list.getLength(); i++) {
-            Node node = list.item(i);
-
-            // parse variable element
-            if (node.getNodeName().equals("variable")) {
-                NodeList children = node.getChildNodes();
-
-                // create Variable Object
-                String variableName = ((Element) node).getAttribute("name");
-                Variable variable = new Variable(variableName);
-
-                // loop through to fill Variable
-                String type = null, value = null;
-                for (int j = 0; j < children.getLength(); j++) {
-                    Node child = children.item(j);
-
-                    // get the Variable's name
-                    if (child.getNodeName().equals("type")) {
-                        type = XMLUtils.getSimpleElementText((Element) child,
-                                true).toLowerCase();
-
-                        // get the Variable's value
-                    } else if (child.getNodeName().equals("value")) {
-                        value = PathUtils.doDynamicReplacement(XMLUtils
-                                .getSimpleElementText((Element) child, false));
-
-                        // get the Variable's value's precision infomation
-                    } else if (child.getNodeName().equals("precision")) {
-                        NodeList grandChildren = child.getChildNodes();
-                        for (int k = 0; k < grandChildren.getLength(); k++) {
-                            Node grandChild = grandChildren.item(k);
-                            // get the precision
-                            if (grandChild.getNodeName().equals("locations")) {
-                                variable.setPrecision(Integer.parseInt(XMLUtils
-                                        .getSimpleElementText((Element) grandChild, true)));
-                                // get the fill character to meet the precision
-                                // [optional]
-                            } else if (grandChild.getNodeName().equals("fill")) {
-                                variable.setFillString(
-                                    XMLUtils.getSimpleElementText((Element) grandChild, false));
-                                // get the side for which the fill character
-                                // will be applied [optional]
-                            } else if (grandChild.getNodeName().equals("side")) {
-                                variable.setFillSide(
-                                        (XMLUtils.getSimpleElementText((Element) grandChild, true)
-                                            .toLowerCase().equals("front")) 
-                                                ? variable.FILL_FRONT
-                                                : variable.FILL_BACK);
-                            }
-                        }
-                    }
-                }
-                // determine if variable is an Integer or a String
-                if (type.equals("int")) {
-                    variable.setValue(Integer.valueOf(value));
-                } else {
-                  variable.setValue(value);
-                }
-
-                // store Variable in list of Variables
-                GlobalVariables.ConcurrentHashMap.put(variable.getName(), variable);
-            }
-        }
-    }
-
-    private void parseMethodsXML(NodeList list) {
-
-        // loop though all method elements
-        for (int i = 0; i < list.getLength(); i++) {
-            Node node = list.item(i);
-            // parse method element
-            if (node.getNodeName().equals("method")) {
-                NodeList children = node.getChildNodes();
-
-                // create Method Object
-                String methodName = ((Element) node).getAttribute("name");
-                Method method = new Method(methodName);
-
-                // loop through to fill Method Object
-                for (int j = 0; j < children.getLength(); j++) {
-                    Node child = children.item(j);
-
-                    // get the Method's behavoir
-                    if (child.getNodeName().equals("action")) {
-                        method.setBehavoir(XMLUtils.getSimpleElementText(
-                                (Element) child, false));
-
-                        // get the Method's arguments
-                    } else if (child.getNodeName().equals("args")) {
-                        String name, argType = null;
-                        NodeList grandChildren = child.getChildNodes();
-
-                        // loop for every arg element
-                        for (int k = 0; k < grandChildren.getLength(); k++) {
-                            Node grandChild = grandChildren.item(k);
-
-                            // parse arg element
-                            if (grandChild.getNodeName().equals("arg")) {
-                                name = ((Element) grandChild)
-                                        .getAttribute("name");
-
-                                // get arg element properties
-                                NodeList greatGrandChildren = grandChild
-                                        .getChildNodes();
-                                for (int l = 0; l < greatGrandChildren
-                                        .getLength(); l++) {
-                                    Node greatGrandChild = greatGrandChildren
-                                            .item(l);
-                                    if (greatGrandChild.getNodeName().equals(
-                                            "type")) {
-                                        argType = XMLUtils.getSimpleElementText(
-                                                (Element) greatGrandChild,
-                                                true);
-                                    }
-                                }
-
-                                // create argument signature in Method
-                                method
-                                        .addArgSignature(
-                                                name,
-                                                ((argType.toLowerCase()
-                                                        .equals("int")) ? method.INT
-                                                        : method.STRING));
-                            }
-                        }
-                    }
-                }
-
-                // store Method in list of Methods
-                methodRepo.put(method.getName(), method);
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/FileListParser.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/FileListParser.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/FileListParser.java
deleted file mode 100644
index d540cf0..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/FileListParser.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.filerestrictions.parsers;
-
-//JDK imports
-import java.io.FileInputStream;
-import java.util.Scanner;
-
-
-import org.apache.oodt.cas.metadata.Metadata;
-//OODT imports
-import org.apache.oodt.cas.pushpull.exceptions.ParserException;
-import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFile;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFileStructure;
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- *
- * <p>Describe your class here</p>.
- */
-public class FileListParser implements Parser {
-
-	public FileListParser() {}
-	
-    public VirtualFileStructure parse(FileInputStream inputFile, Metadata metadata)
-            throws ParserException {
-        Scanner scanner = new Scanner(inputFile);
-        VirtualFile root = VirtualFile.createRootDir();
-        String initialCdDir = "/";
-        if (scanner.hasNextLine()) {
-            initialCdDir = scanner.nextLine();
-            while (scanner.hasNextLine()) {
-                new VirtualFile(root, initialCdDir + "/" + scanner.nextLine(),
-                        false);
-            }
-        }
-        return new VirtualFileStructure(initialCdDir, root);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/GenericEmailParser.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/GenericEmailParser.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/GenericEmailParser.java
deleted file mode 100644
index 3f74407..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/parsers/GenericEmailParser.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.oodt.cas.pushpull.filerestrictions.parsers;
-
-//OODT imports
-import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFile;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFileStructure;
-import org.apache.oodt.cas.pushpull.exceptions.ParserException;
-
-//Google imports
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-//JDK imports
-import java.io.FileInputStream;
-import java.util.List;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * A generic email parser which generates file paths to be downloaded by using a defined java
- * Pattern. The pattern should specify pattern groups for file paths in the matching pattern.
- * These groups will then be extracted and added to the file structure.
- *
- * @author bfoster@apache.org (Brian Foster)
- */
-public class GenericEmailParser implements Parser {
-
-  private static final Logger log = Logger.getLogger(GenericEmailParser.class.getCanonicalName());
-
-  public static final String FILE_PATTERNS_PROPERTY_NAME =
-      "org.apache.oodt.cas.pushpull.generic.email.parser.file.pattern";
-  public static final String CHECK_FOR_PATTERN_PROPERTY_NAME =
-      "org.apache.oodt.cas.pushpull.generic.email.parser.check.for.pattern";
-  public static final String PATH_TO_ROOT_PROPERTY_NAME =
-      "org.apache.oodt.cas.pushpull.generic.email.parser.path.to.root";
-  public static final String METADATA_KEYS =
-      "org.apache.oodt.cas.pushpull.generic.email.parser.metadata.keys";
-  public static final String METADATA_KEY_PREFIX =
-      "org.apache.oodt.cas.pushpull.generic.email.parser.metadata.";      
-  
-  private final String filePattern;
-  private final String checkForPattern;
-  private final String pathToRoot;
-
-  public GenericEmailParser() {
-    filePattern = loadFilePattern();
-    checkForPattern = loadCheckForPattern();
-    pathToRoot = loadPathToRoot();
-  }
- 
-  public GenericEmailParser(String filePattern, String checkForPattern, String pathToRoot) {
-    this.filePattern = filePattern;
-    this.checkForPattern = checkForPattern;
-    this.pathToRoot = Strings.nullToEmpty(pathToRoot);
-  }
-
-  @Override
-  public VirtualFileStructure parse(FileInputStream emailFile, Metadata metadata)
-      throws ParserException {
-    log.info("GenericEmailParser is parsing email: " + emailFile);
-
-    VirtualFile root = VirtualFile.createRootDir();
-
-    String emailText = readEmail(emailFile);
-    if (!isValidEmail(emailText)) {
-      throw new ParserException("Failed to find check for pattern in email: " + checkForPattern);
-    }
-    List<String> filePaths = generateFilePaths(emailText);
-    readMetadata(emailText, metadata);
-
-    for (String filePath : filePaths) {
-      new VirtualFile(root, pathToRoot + filePath, false);
-    }
-
-    return new VirtualFileStructure("/", root);
-  }
-
-  private String readEmail(FileInputStream emailFile) {
-    StringBuilder emailText = new StringBuilder("");
-    Scanner scanner = new Scanner(emailFile);
-    while (scanner.hasNextLine()) {
-      emailText.append(scanner.nextLine()).append("\n");
-    }
-    scanner.close();
-    return emailText.toString();
-  }
-
-  private List<String> generateFilePaths(String emailText) {
-    List<String> filePaths = Lists.newArrayList();
-    Pattern pattern = Pattern.compile(filePattern);
-    Matcher m = pattern.matcher(emailText);
-    if (m.find()) {
-      // Ignore index 0, as that is the matching string for pattern.
-      for (int i = 1; i <= m.groupCount(); i++) {
-        filePaths.add(m.group(i));          
-      }
-    }
-    return filePaths;
-  }
-
-  private void readMetadata(String emailText, Metadata metadata) {
-    Set<String> metadataKeys = loadMetadataKeys();
-    for (String metadataKey : metadataKeys) {
-      String metadataPattern = loadMetadataKey(metadataKey);
-      if (metadataPattern == null) {
-        log.log(Level.SEVERE, "Failed to load metadata pattern for key: " + metadataKey);
-      } else {
-        Pattern pattern = Pattern.compile(metadataPattern);
-        Matcher m = pattern.matcher(emailText);
-        if (m.find()) {
-          // Ignore index 0, as that is the matching string for pattern.
-          String metadatValue = m.group(1);
-          metadata.replaceMetadata(metadataKey, metadatValue);
-        }
-      }
-    }
-  }
-
-  private boolean isValidEmail(String emailText) {
-    Pattern pattern = Pattern.compile(checkForPattern);
-    Matcher m = pattern.matcher(emailText.replaceAll("\n", " "));
-    return m.find();
-  }
-  
-  private String loadFilePattern() {
-    return System.getProperty(FILE_PATTERNS_PROPERTY_NAME);
-  }
-  
-  private String loadCheckForPattern() {
-    return System.getProperty(CHECK_FOR_PATTERN_PROPERTY_NAME);    
-  }
-  
-  private String loadPathToRoot() {
-    return Strings.nullToEmpty(System.getProperty(PATH_TO_ROOT_PROPERTY_NAME));
-  }
-
-  private Set<String> loadMetadataKeys() {
-    return Sets.newHashSet(Splitter.on(",").omitEmptyStrings().split(
-        Strings.nullToEmpty(System.getProperty(METADATA_KEYS))));
-  }
-
-  private String loadMetadataKey(String key) {
-    return System.getProperty(METADATA_KEY_PREFIX + key);
-  }
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java
deleted file mode 100644
index 753e09c..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/filerestrictions/renamingconventions/RenamingConvention.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.cas.pushpull.filerestrictions.renamingconventions;
-
-//JDK imports
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-//OODT imports
-import org.apache.oodt.cas.metadata.util.PathUtils;
-import org.apache.oodt.cas.pushpull.retrievalsystem.RemoteFile;
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- * 
- * <p>
- * Describe your class here
- * </p>.
- */
-public class RenamingConvention {
-	
-	private static Logger LOG = Logger.getLogger(RenamingConvention.class.getName());
-
-    private RenamingConvention() throws InstantiationException {
-        throw new InstantiationException("Don't construct RenamingConventions!");
-    }
-
-    /**
-     * Generates a unique file name for the given ProtocolFile
-     * 
-     * @param fileToGenNewNameFor
-     *            The file for which a unique name will be generated
-     * @return The unique file name (just the name).
-     */
-    public static String rename(RemoteFile fileToGenNewNameFor,
-            String renamingString) {
-    	try {
-    	    renamingString = grepReplace(renamingString, fileToGenNewNameFor);
-    	    renamingString = grepRemoveReplace(renamingString, fileToGenNewNameFor);
-	        renamingString = replace(renamingString, "[FILENAME]",
-	                fileToGenNewNameFor.getProtocolFile().getName());
-	        renamingString = replace(renamingString, "[PATH_NO_FILENAME]",
-	                getParentPath(fileToGenNewNameFor));
-	        renamingString = replace(renamingString, "[HOST]", fileToGenNewNameFor
-	                .getProtocolFile().getSite().getURL().getHost());
-	        renamingString = replace(renamingString, "[PARENT_FILENAME]",
-	                getParentFileName(fileToGenNewNameFor));
-	        renamingString = replace(renamingString, "[PARENT_PATH_NO_FILENAME]",
-	                getGrandParentPath(fileToGenNewNameFor));
-	        renamingString = replace(renamingString, "[URL]", fileToGenNewNameFor
-	                .getProtocolFile().getSite().getURL().toExternalForm());
-	        renamingString = replace(renamingString, "[IS_DIR]", String
-	                .valueOf(fileToGenNewNameFor.getProtocolFile().isDir()));
-	        renamingString = PathUtils.doDynamicReplacement(
-	            renamingString, fileToGenNewNameFor.getAllMetadata());
-    	}catch (Exception e) {
-    		LOG.log(Level.WARNING, "Failed to rename " + fileToGenNewNameFor 
-    				+ " : " + e.getMessage());
-    	}
-        return renamingString;
-    }
-
-    private static String grepReplace(String theString, RemoteFile fileToGenNewNameFor) {
-        Pattern grepPattern = Pattern.compile("\\[GREP\\(.*\\,.*\\)\\]");
-        Matcher grepMatcher = grepPattern.matcher(theString);
-        while (grepMatcher.find()) {
-            String origGrepString = theString.substring(grepMatcher.start(),
-                    grepMatcher.end()).trim();
-            String grepString = origGrepString.replace("[GREP('", "").replace(
-                    "')]", "").trim();
-            String[] grepStringSplit = grepString.split("','");
-            String pattern = grepStringSplit[0];
-            String string = rename(fileToGenNewNameFor, grepStringSplit[1]);
-            Pattern p = Pattern.compile(pattern);
-            Matcher m = p.matcher(string);
-            if (m.find()) {
-                theString = theString.replace(origGrepString, string.substring(
-                        m.start(), m.end()));
-            } else {
-                theString = theString.replace(origGrepString, "null");
-            }
-        }
-        return theString;
-    }
-
-    private static String grepRemoveReplace(String theString, RemoteFile fileToGenNewNameFor) {
-        Pattern grepPattern = Pattern.compile("\\[GREP_RM\\(.*,.*\\)\\]");
-        Matcher grepMatcher = grepPattern.matcher(theString);
-        while (grepMatcher.find()) {
-            String origGrepString = theString.substring(grepMatcher.start(),
-                    grepMatcher.end()).trim();
-            String grepString = origGrepString.replace("[GREP_RM('", "")
-                    .replace("')]", "").trim();
-            String[] grepStringSplit = grepString.split("','");
-            String pattern = grepStringSplit[0];
-            String string = rename(fileToGenNewNameFor, grepStringSplit[1]);
-            System.out.println("PAT_STR: " + pattern + " " + string);
-            Pattern p = Pattern.compile(pattern);
-            Matcher m = p.matcher(string);
-            if (m.find()) {
-                theString = theString.replace(origGrepString, string.replace(
-                        string.substring(m.start(), m.end()), ""));
-            } else {
-                theString = theString.replace(origGrepString, "null");
-            }
-        }
-        return theString;
-    }
-     
-    private static String replace(String theString,
-            String theValueToBeReplaced, String whatToReplaceWith) {
-        if (theValueToBeReplaced == null || theValueToBeReplaced.equals("")) {
-            return theString;
-        }
-        if (whatToReplaceWith == null) {
-            whatToReplaceWith = "";
-        }
-        return theString.replace(theValueToBeReplaced, whatToReplaceWith);
-    }
-
-    private static String getParentPath(RemoteFile fileToGenNewNameFor) {
-        String parentPath = "";
-        try {
-            parentPath = fileToGenNewNameFor.getProtocolFile().getParent().getPath();
-        } catch (Exception ignored) {
-        }
-        return parentPath;
-    }
-
-    private static String getParentFileName(RemoteFile fileToGenNewNameFor) {
-        String parentFileName = "";
-        try {
-            parentFileName = fileToGenNewNameFor.getProtocolFile().getParent().getName();
-        } catch (Exception ignored) {
-        }
-        return parentFileName;
-    }
-
-    private static String getGrandParentPath(RemoteFile fileToGenNewNameFor) {
-        String grandParentPath = "";
-        try {
-            grandParentPath = fileToGenNewNameFor.getProtocolFile().getParent()
-                    .getParent().getPath();
-        } catch (Exception ignored) {
-        }
-        return grandParentPath;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/objectfactory/PushPullObjectFactory.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/objectfactory/PushPullObjectFactory.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/objectfactory/PushPullObjectFactory.java
deleted file mode 100644
index 2e9d9af..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/objectfactory/PushPullObjectFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.objectfactory;
-
-//OODT imports
-import org.apache.oodt.cas.filemgr.ingest.Cache;
-import org.apache.oodt.cas.filemgr.ingest.CacheFactory;
-import org.apache.oodt.cas.filemgr.ingest.Ingester;
-
-//JDK imports
-import java.lang.reflect.InvocationTargetException;
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- * 
- * <p>
- * Describe your class here
- * </p>.
- */
-public class PushPullObjectFactory {
-
-    private PushPullObjectFactory() throws InstantiationException {
-        throw new InstantiationException("Don't construct factory classes!");
-    }
-
-    public static <T> T createNewInstance(Class<T> clazz) throws InstantiationException {
-    	try {
-			return clazz.newInstance();
-		} catch (Exception e) {
-            throw new InstantiationException(
-                    "Failed to create new object : "
-                            + e.getMessage());
-		}
-    }
-    
-    public static Ingester createIngester(String ingesterClass,
-            String cacheFactoryClass) throws InstantiationException,
-            IllegalAccessException, ClassNotFoundException,
-            IllegalArgumentException, SecurityException,
-            InvocationTargetException, NoSuchMethodException {
-        String dataTransferFactory = System
-                .getProperty("org.apache.oodt.cas.filemgr.datatransfer.factory");
-        System.out.println("TRANSFER: " + dataTransferFactory);
-        if (cacheFactoryClass == null || cacheFactoryClass.equals("")) {
-            return (Ingester) Class.forName(ingesterClass).getConstructor(
-                    dataTransferFactory.getClass()).newInstance(
-                    dataTransferFactory);
-        } else {
-            Class<CacheFactory> cacheFactory = (Class<CacheFactory>) Class
-                    .forName(cacheFactoryClass);
-            Cache cache = cacheFactory.newInstance().createCache();
-            return (Ingester) Class.forName(ingesterClass).getConstructor(
-                    dataTransferFactory.getClass(), cache.getClass())
-                    .newInstance(dataTransferFactory, cache);
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
deleted file mode 100644
index ccafbbf..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
+++ /dev/null
@@ -1,603 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.cas.pushpull.protocol;
-
-//OODT imports
-
-import org.apache.oodt.cas.protocol.Protocol;
-import org.apache.oodt.cas.protocol.ProtocolFactory;
-import org.apache.oodt.cas.protocol.ProtocolFile;
-import org.apache.oodt.cas.protocol.auth.BasicAuthentication;
-import org.apache.oodt.cas.protocol.exceptions.ProtocolException;
-import org.apache.oodt.cas.protocol.util.ProtocolFileFilter;
-import org.apache.oodt.cas.pushpull.config.ProtocolInfo;
-import org.apache.oodt.cas.pushpull.exceptions.RemoteConnectionException;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.Vector;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-//JDK imports
-
-
-/**
- * This class is responsible for creating the appropriate Protocol for the given
- * RemoteSites. The boolean argument 'allowReuse' allows for one unique protocol
- * for each URL. That is, if allowReuse is set to true, then if no Protocol has
- * been created for the current site, the Protocol created will be saved and
- * then returned for any later called with allowReuse equals true. This is to
- * allow for the same Protocol object to be used by several classes. The
- * Protocol class has been synchronized so this is thread-safe. If you set
- * 'allowReuse' to false then a new Protocol object will be created and
- * returned.<br>
- * <br>
- *
- * @author bfoster
- */
-public class ProtocolHandler {
-
-  private final ConcurrentHashMap<URI, ProtocolFactory> urlAndProtocolFactory;
-
-  private final ConcurrentHashMap<URI, Protocol> reuseProtocols;
-
-  private final ConcurrentHashMap<RemoteSiteFile, PagingInfo> pageInfos;
-
-  private final ConcurrentHashMap<RemoteSiteFile, List<RemoteSiteFile>> pathAndFileListMap;
-
-  private final ProtocolInfo pi;
-
-  private static final Logger LOG = Logger.getLogger(ProtocolHandler.class
-      .getName());
-
-  /**
-   * Creates a new ProtocolHandler for the given Config object
-   *
-   * @param pi
-   *          The Config object that guides this ProtocolHandler in making class
-   *          instanciations
-   */
-  public ProtocolHandler(ProtocolInfo pi) {
-    this.pi = pi;
-    urlAndProtocolFactory = new ConcurrentHashMap<URI, ProtocolFactory>();
-    reuseProtocols = new ConcurrentHashMap<URI, Protocol>();
-    pageInfos = new ConcurrentHashMap<RemoteSiteFile, PagingInfo>();
-    pathAndFileListMap = new ConcurrentHashMap<RemoteSiteFile, List<RemoteSiteFile>>();
-  }
-
-  /**
-   * Returns the appropriate protocol for the given Path
-   *
-   * @param pFile
-   *          Used to determine the appropriate Protocol to be returned and the
-   *          path to navigate on if navigateToPathLoc is set to true.
-   * @param allowReuse
-   *          Set to true if you would like ProtocolHandler to take care of the
-   *          protocol returned (i.e. reuseable protocols may be returned by
-   *          this method again, if it is the appropriate protocol type for a
-   *          given Path. Also ProtocolHandler will take care of disconnecting
-   *          the reuseable protocols)
-   * @param navigateToPathLoc
-   *          If true, will navigate the to the end of the Path location
-   *          specified
-   * @return Protocol for the given Path
-   * @throws RemoteConnectionException
-   *           If there is an error creating the protocol
-   */
-  public Protocol getAppropriateProtocol(RemoteSiteFile pFile,
-      boolean allowReuse, boolean navigateToPathLoc)
-      throws RemoteConnectionException {
-    try {
-      Protocol protocol = getAppropriateProtocol(pFile, allowReuse);
-      if (protocol != null && navigateToPathLoc) {
-        if (pFile.isDir()) {
-          this.cd(protocol, pFile);
-        } else if (pFile.getParent() != null) {
-          this.cd(protocol, new RemoteSiteFile(pFile.getParent(), pFile.getSite()));
-        }
-      }
-      return protocol;
-    } catch (Exception e) {
-      throw new RemoteConnectionException(
-          "Failed to get appropriate protocol for " + pFile + " : "
-              + e.getMessage(), e);
-    }
-  }
-
-  private Protocol getAppropriateProtocol(RemoteSiteFile pFile,
-      boolean allowReuse) throws ProtocolException {
-    return this.getAppropriateProtocolBySite(pFile.getSite(), allowReuse);
-  }
-
-  public Protocol getAppropriateProtocolBySite(RemoteSite remoteSite,
-      boolean allowReuse) throws ProtocolException {
-    Protocol protocol = null;
-    try {
-      if ((allowReuse && ((protocol = reuseProtocols.get(remoteSite.getURL().toURI())) == null))
-          || !allowReuse) {
-        ProtocolFactory protocolFactory = null;
-        try {
-          protocolFactory = this.urlAndProtocolFactory
-              .get(remoteSite.getURL().toURI());
-        } catch (URISyntaxException e) {
-          LOG.log(Level.SEVERE, "could not convert url to uri: Message: " + e.getMessage());
-        }
-        if (protocolFactory == null) {
-          LinkedList<Class<ProtocolFactory>> protocolClasses = pi
-              .getProtocolClassesForProtocolType(remoteSite.getURL()
-                                                           .getProtocol());
-          for (Class<ProtocolFactory> clazz : protocolClasses) {
-            try {
-              if ((protocol = (protocolFactory = clazz.newInstance())
-                  .newInstance()) != null) {
-                if (!connect(protocol, remoteSite, true)) {
-                  LOG.log(
-                      Level.WARNING,
-                      "ProtocolFactory "
-                          + protocolFactory.getClass().getCanonicalName()
-                          + " is not compatible with server at "
-                          + remoteSite.getURL());
-                  protocol = null;
-                } else {
-                  this.urlAndProtocolFactory.put(remoteSite.getURL().toURI(),
-                      protocolFactory);
-                  break;
-                }
-              }
-            } catch (Exception e) {
-              LOG.log(Level.WARNING, "Failed to instanciate protocol " + clazz
-                  + " for " + remoteSite.getURL());
-            }
-          }
-          if (protocol == null) {
-            throw new ProtocolException("Failed to get appropriate protocol for "
-                                        + remoteSite);
-          }
-        } else {
-          connect(protocol = protocolFactory.newInstance(), remoteSite, false);
-        }
-        if (allowReuse) {
-          try {
-            this.reuseProtocols.put(remoteSite.getURL().toURI(), protocol);
-          } catch (URISyntaxException e) {
-            LOG.log(Level.SEVERE, "Couildn't covert URL to URI Mesage: " + e.getMessage());
-          }
-        }
-      }
-    } catch (URISyntaxException e) {
-      LOG.log(Level.SEVERE, "could not convert url to uri: Message: "+e.getMessage());
-    }
-    return protocol;
-  }
-
-  public synchronized List<RemoteSiteFile> nextPage(RemoteSite site, Protocol protocol)
-      throws RemoteConnectionException, ProtocolException {
-    return nextPage(site, protocol, null);
-  }
-
-  /**
-   * @param protocol
-   * @return
-   * @throws RemoteConnectionException
-   * @throws ProtocolException
-   */
-  public synchronized List<RemoteSiteFile> nextPage(RemoteSite site, Protocol protocol,
-      ProtocolFileFilter filter) throws RemoteConnectionException,
-      ProtocolException {
-
-    PagingInfo pgInfo = this.getPagingInfo(this.pwd(site, protocol));
-    try {
-      System.out.println("PageSize: " + pi.getPageSize() + " PageLoc: "
-          + pgInfo.getPageLoc());
-      List<RemoteSiteFile> fileList = this.ls(site, protocol);
-      System.out.println("FileList size: " + fileList.size());
-
-      if (this.getDynamicFileList(site, protocol) == null
-          && !this.passesDynamicDetection(pgInfo, fileList)) {
-        LOG.log(
-            Level.SEVERE,
-            "Remote directory '"
-                + this.pwd(site, protocol)
-                + "' file list size has changed -- setting directory as dynamic and resetting page location");
-        this.putDynamicFileList(site, protocol, fileList);
-        pgInfo.updatePageInfo(0, fileList);
-      }
-
-      List<RemoteSiteFile> page = new LinkedList<RemoteSiteFile>();
-      int curLoc = pgInfo.getPageLoc();
-      for (; page.size() < pi.getPageSize() && curLoc < fileList.size(); curLoc++) {
-        if (filter == null || filter.accept(fileList.get(curLoc))) {
-          page.add(fileList.get(curLoc));
-        }
-      }
-      pgInfo.updatePageInfo(curLoc, fileList);
-
-      return page;
-    } catch (Exception e) {
-      LOG.log(Level.SEVERE, e.getMessage());
-      throw new RemoteConnectionException(
-          "Failed getting next page for protocol " + protocol + "-- pgStart = "
-              + pgInfo.getPageLoc() + " pgSize = " + pi.getPageSize() + " : "
-              + e.getMessage());
-    }
-
-  }
-
-  private boolean passesDynamicDetection(PagingInfo pgInfo,
-      List<RemoteSiteFile> newLS) {
-    return !(pgInfo.getSizeOfLastLS() != -1
-             && (pgInfo.getSizeOfLastLS() != newLS.size() || (newLS.size() != 0
-                                                              && pgInfo.getPageLoc() < newLS.size() && (newLS.get(pgInfo
-        .getPageLoc()) == null || !newLS.get(pgInfo.getPageLoc()).equals(
-        pgInfo.getRemoteSiteFileAtPageLoc())))));
-  }
-
-  public void download(Protocol protocol, RemoteSiteFile fromFile, File toFile,
-      boolean delete) throws RemoteConnectionException {
-
-    // rename file for download
-    File downloadFile = new File(
-        String.format("%s/Downloading_%s", toFile.getParent(), toFile.getName()));
-    toFile.renameTo(downloadFile);
-
-    LOG.log(Level.INFO, "Starting to download " + fromFile);
-    try {
-      // try to download the file
-      protocol.get(fromFile, downloadFile);
-
-      // rename file back to original name
-      if (!downloadFile.renameTo(toFile)) {
-        throw new IOException(
-            String.format("Failed to rename file %s to %s", downloadFile, toFile));
-      }
-
-      // delete file is specified
-      if (delete) {
-        if (!this.delete(protocol, fromFile)) {
-          LOG.log(Level.WARNING, "Failed to delete file '" + fromFile
-                                 + "' from server '" + fromFile.getSite() + "'");
-        } else {
-          LOG.log(Level.INFO, "Successfully deleted file '" + fromFile
-                              + "' from server '" + fromFile.getSite() + "'");
-        }
-      }
-
-      LOG.log(Level.INFO, "Finished downloading " + fromFile + " to " + toFile);
-
-    } catch (Exception e) {
-      downloadFile.delete();
-      throw new RemoteConnectionException("Failed to download file " + fromFile
-          + " : " + e.getMessage(), e);
-    }
-  }
-
-  /**
-   * Connects the given Protocol to the given URL
-   *
-   * @param protocol
-   *          The Protocol that will be connected
-   * @param remoteSite
-   *          The server to which the Protocol will connect
-   * @param test
-   */
-  public boolean connect(Protocol protocol, RemoteSite remoteSite, boolean test) {
-    for (int tries = 0; tries < 3; tries++) {
-
-      // wait for 5 secs before next retry
-      if (tries > 0) {
-        LOG.log(Level.INFO, "Will retry connecting to " + remoteSite
-            + " in 5 seconds");
-        synchronized (this) {
-          try {
-            System.out.print("Waiting");
-            for (int i = 0; i < 5; i++) {
-              System.out.print(" .");
-              wait(1000);
-            }
-            System.out.println();
-          } catch (Exception ignored) {
-          }
-        }
-      }
-
-      try {
-        // make sure protocol is disconnected
-        try {
-          protocol.close();
-        } catch (Exception ignored) {
-        }
-
-        // try connecting Protocol
-        protocol.connect(
-            remoteSite.getURL().getHost(),
-            new BasicAuthentication(remoteSite.getUsername(), remoteSite
-                .getPassword()));
-
-        // check connection
-        if (protocol.connected()
-            && (!test || isOkProtocol(protocol, remoteSite))) {
-          LOG.log(Level.INFO,
-              "Successfully connected to " + remoteSite.getURL()
-                  + " with protocol '" + protocol.getClass().getCanonicalName()
-                  + "' and username '" + remoteSite.getUsername() + "'");
-          return true;
-        } else {
-          return false;
-        }
-
-      } catch (Exception e) {
-        LOG.log(Level.WARNING, "Error occurred while connecting to "
-            + remoteSite + " : " + e.getMessage(), e);
-      }
-
-    }
-    return false;
-  }
-
-  private boolean isOkProtocol(Protocol protocol, RemoteSite remoteSite) {
-    try {
-      LOG.log(Level.INFO, "Testing protocol "
-          + protocol.getClass().getCanonicalName()
-          + " . . . this may take a few minutes . . .");
-      // test ls, cd, and pwd
-      this.cdToHOME(protocol);
-      RemoteSiteFile home = this.pwd(remoteSite, protocol);
-      this.ls(remoteSite, protocol);
-      if (remoteSite.getCdTestDir() != null) {
-        this.cd(protocol, new RemoteSiteFile(home, remoteSite.getCdTestDir(),
-            true, remoteSite));
-      } else {
-        this.cdToROOT(protocol);
-      }
-      this.cdToHOME(protocol);
-      if (home == null || !home.equals(this.pwd(remoteSite, protocol))) {
-        throw new ProtocolException("Home directory not the same after cd");
-      }
-    } catch (Exception e) {
-      LOG.log(Level.SEVERE, "Protocol "
-          + protocol.getClass().getCanonicalName()
-          + " failed compatibility test : " + e.getMessage(), e);
-      return false;
-    }
-    return true;
-  }
-
-  public void cdToROOT(Protocol protocol) throws ProtocolException {
-    protocol.cdRoot();
-  }
-
-  public void cdToHOME(Protocol protocol) throws ProtocolException {
-    protocol.cdHome();
-  }
-
-  public boolean isProtocolConnected(Protocol protocol) {
-    return protocol.connected();
-  }
-
-  public void cd(Protocol protocol, RemoteSiteFile file)
-      throws ProtocolException {
-    protocol.cd(file);
-  }
-
-  public RemoteSiteFile getProtocolFileFor(RemoteSite site, Protocol protocol, String file,
-      boolean isDir) throws ProtocolException {
-    return this.getProtocolFileByProtocol(site, protocol, file, isDir);
-  }
-
-  public synchronized boolean delete(Protocol protocol, RemoteSiteFile file) {
-    try {
-      PagingInfo pgInfo = this.getPagingInfo(file.getRemoteParent());
-      List<RemoteSiteFile> fileList = this.ls(protocol, file.getRemoteParent());
-      int indexOfFile = fileList.indexOf(file);
-      if (indexOfFile != -1) {
-        protocol.delete(file);
-        fileList.remove(indexOfFile);
-        System.out.println("IndexOfFile: " + indexOfFile + " PageIndex: "
-            + pgInfo.getPageLoc());
-        if (indexOfFile < pgInfo.getPageLoc()
-            || indexOfFile == fileList.size() - 1) {
-          pgInfo.updatePageInfo(pgInfo.getPageLoc() - 1, fileList);
-        } else {
-          pgInfo.updatePageInfo(pgInfo.getPageLoc(), fileList);
-        }
-        return true;
-      } else {
-        return false;
-      }
-    } catch (Exception e) {
-      LOG.log(Level.SEVERE, "Failed to delete file", e);
-      return false;
-    }
-  }
-
-  private synchronized void putPgInfo(PagingInfo pgInfo, RemoteSiteFile pFile) {
-    this.pageInfos.put(pFile, pgInfo);
-  }
-
-  private synchronized PagingInfo getPagingInfo(RemoteSiteFile pFile) {
-    PagingInfo pgInfo = this.pageInfos.get(pFile);
-    if (pgInfo == null) {
-      this.putPgInfo(pgInfo = new PagingInfo(), pFile);
-    }
-    return pgInfo;
-  }
-
-  public RemoteSiteFile pwd(RemoteSite site, Protocol protocol) throws ProtocolException {
-    return new RemoteSiteFile(protocol.pwd(), site);
-  }
-
-  public List<RemoteSiteFile> ls(Protocol protocol, RemoteSiteFile dir)
-      throws ProtocolException {
-    List<RemoteSiteFile> fileList = this.getDynamicFileList(dir.getSite(), protocol);
-    if (fileList == null) {
-      protocol.cd(dir);
-      fileList = toRemoteSiteFiles(protocol.ls(), dir.getSite());
-    }
-    return fileList;
-  }
-
-  public List<RemoteSiteFile> ls(RemoteSite site, Protocol protocol) throws ProtocolException {
-    List<RemoteSiteFile> fileList = this.getDynamicFileList(site, protocol);
-    if (fileList == null) {
-      fileList = toRemoteSiteFiles(protocol.ls(), site);
-    }
-    return fileList;
-  }
-
-  public List<RemoteSiteFile> ls(RemoteSite site, Protocol protocol, ProtocolFileFilter filter)
-      throws ProtocolException {
-    List<RemoteSiteFile> fileList = this.getDynamicFileList(site, protocol);
-    if (fileList == null) {
-      fileList = toRemoteSiteFiles(protocol.ls(filter), site);
-    }
-    return fileList;
-  }
-
-  private synchronized List<RemoteSiteFile> getDynamicFileList(RemoteSite site, Protocol protocol)
-      throws ProtocolException {
-    return (List<RemoteSiteFile>) (List<?>) this.pathAndFileListMap.get(this
-        .pwd(site, protocol));
-  }
-
-  private synchronized void putDynamicFileList(RemoteSite site, Protocol protocol,
-      List<RemoteSiteFile> fileList) throws ProtocolException {
-    this.pathAndFileListMap.put(this.pwd(site, protocol), fileList);
-  }
-
-  public synchronized RemoteSiteFile getHomeDir(RemoteSite site, Protocol protocol) {
-    try {
-      protocol.cdHome();
-      return new RemoteSiteFile(protocol.pwd(), site);
-    } catch (Exception e) {
-      LOG.log(Level.SEVERE, e.getMessage());
-      return null;
-    }
-  }
-
-  public String getAbsPathFor(Protocol protocol, String path, boolean isDir) {
-    try {
-      protocol.cd(new ProtocolFile(path, isDir));
-      return protocol.pwd().getAbsoluteFile().getPath();
-    } catch (Exception e) {
-      LOG.log(Level.SEVERE, e.getMessage());
-      return null;
-    }
-  }
-
-  /**
-   * Disconnects and logs out the given Protocol
-   *
-   * @param protocol
-   *          The Protocol to be logout out and disconnected
-   * @throws RemoteConnectionException
-   */
-  public void disconnect(Protocol protocol) throws RemoteConnectionException {
-    try {
-      LOG.log(Level.INFO, "Disconnecting protocol " + protocol.getClass().getName());
-      protocol.close();
-    } catch (Exception e) {
-      throw new RemoteConnectionException("Error disconnecting " + protocol.getClass().getName()
-          + " : " + e.getMessage());
-    }
-  }
-
-  /**
-   * Disconnects all waiting Protocols and clears the waiting lists. Also clears
-   * the current Protocol
-   *
-   * @throws RemoteConnectionException
-   */
-  public void close() throws RemoteConnectionException {
-    Set<Entry<URI, Protocol>> entries = reuseProtocols.entrySet();
-    for (Entry<URI, Protocol> entry : entries) {
-      disconnect(entry.getValue());
-    }
-    this.reuseProtocols.clear();
-    this.urlAndProtocolFactory.clear();
-    this.pageInfos.clear();
-    this.pathAndFileListMap.clear();
-  }
-
-  private synchronized RemoteSiteFile getProtocolFileByProtocol(
-      RemoteSite site, Protocol protocol, String file, boolean isDir) throws ProtocolException {
-    try {
-      if (!file.startsWith("/")) {
-        protocol.cdHome();
-        file = protocol.pwd().getPath() + "/" + file;
-      }
-      return new RemoteSiteFile(file, isDir, site);
-    } catch (Exception e) {
-      throw new ProtocolException("Failed to create protocol for " + file
-          + " : " + e.getMessage());
-    }
-  }
-
-  private List<RemoteSiteFile> toRemoteSiteFiles(List<ProtocolFile> files, RemoteSite site) {
-    List<RemoteSiteFile> newFiles = new Vector<RemoteSiteFile>();
-    if (files != null) {
-      for (ProtocolFile file : files) {
-        newFiles.add(new RemoteSiteFile(file, site));
-      }
-    }
-    return newFiles;
-  }
-
-  class PagingInfo {
-
-    private int pageLoc;
-
-    private int sizeOfLastLS;
-
-    private RemoteSiteFile pFileAtPageLoc;
-
-    PagingInfo() {
-      this.pageLoc = 0;
-      this.sizeOfLastLS = -1;
-      this.pFileAtPageLoc = null;
-    }
-
-    synchronized void updatePageInfo(int newPageLoc, List<RemoteSiteFile> ls) {
-      this.sizeOfLastLS = ls.size();
-      this.pageLoc = newPageLoc < 0 ? 0 : newPageLoc;
-      this.pFileAtPageLoc = (this.sizeOfLastLS > 0 && newPageLoc < ls.size()) ? ls
-          .get(newPageLoc) : null;
-    }
-
-    synchronized int getPageLoc() {
-      return this.pageLoc;
-    }
-
-    synchronized int getSizeOfLastLS() {
-      return this.sizeOfLastLS;
-    }
-
-    synchronized RemoteSiteFile getRemoteSiteFileAtPageLoc() {
-      return this.pFileAtPageLoc;
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolPath.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolPath.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolPath.java
deleted file mode 100644
index 9da66db..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolPath.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.protocol;
-
-//JDK imports
-import java.io.Serializable;
-
-/**
- * This class wraps up a URL for a given path, and whether the path specified by
- * the URL is a dirctory or not
- * 
- * @author bfoster
- */
-public class ProtocolPath implements Serializable {
-
-    private static final long serialVersionUID = 807275248811949120L;
-
-    /**
-     * The string verion of the path in the URL
-     */
-    protected String path;
-
-    protected String remotePath;
-
-    protected boolean relativeToHOME;
-
-    /**
-     * Specifies whether this path is a path to a directory or file
-     */
-    protected boolean isDir;
-
-    public ProtocolPath() {
-        super();
-    }
-
-    public ProtocolPath(String path, boolean isDir) {
-        this.isDir = isDir;
-        this.path = this.checkForDelimiters(path);
-    }
-
-    protected String checkForDelimiters(String path) {
-        if (path.endsWith("/") && path.length() > 1) {
-            path = path.substring(0, path.length() - 1);
-        }
-        relativeToHOME = !path.startsWith("/");
-        return path;
-    }
-
-    public boolean isRelativeToHOME() {
-        return relativeToHOME;
-    }
-
-    public String getPathString() {
-        return path;
-    }
-
-    /**
-     * Return the name of the file for which this path belongs
-     * 
-     * @return The Path file name.
-     */
-    public String getFileName() {
-        return path.substring(path.lastIndexOf("/") + 1);
-    }
-
-    /**
-     * Returns the path that is used when downloading the file
-     * 
-     * @return The downloading path
-     */
-    public String getDownloadPath() {
-        return (isDirectory()) ? path : path
-                .substring(0, path.lastIndexOf("/"))
-                + "/" + getDownloadFileName();
-    }
-
-    /**
-     * Returns the file name that is used when downloading the file
-     * 
-     * @return The name used during downloading.
-     */
-    public String getDownloadFileName() {
-        return "Downloading_" + getFileName();
-    }
-
-    /**
-     * Tells whether this path is a path to a directory or not
-     * 
-     * @return True if this Path is a directory
-     */
-    public boolean isDirectory() {
-        return isDir;
-    }
-
-    public boolean equals(Object path) {
-        if (path instanceof ProtocolPath) {
-            ProtocolPath p = (ProtocolPath) path;
-            return (p.getPathString().equals(this.getPathString()));
-        }
-        return false;
-    }
-
-    public String getParentDirPath() {
-        if (path.length() <= 1) {
-            return null;
-        }
-        return path.substring(0, path.lastIndexOf("/"));
-    }
-
-    public String toString() {
-        return (path + " isDir=" + this.isDir);
-    }
-
-    public ProtocolPath getParentPath() {
-        return new ProtocolPath(path.substring(0, path.lastIndexOf("/")), true);
-    }
-
-    @Override
-    public int hashCode() {
-        int result = path != null ? path.hashCode() : 0;
-        result = 31 * result + (remotePath != null ? remotePath.hashCode() : 0);
-        result = 31 * result + (relativeToHOME ? 1 : 0);
-        result = 31 * result + (isDir ? 1 : 0);
-        return result;
-    }
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java
deleted file mode 100644
index 15f5e9a..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSite.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.protocol;
-
-//JDK imports
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- * 
- * <p>
- * Describe your class here
- * </p>.
- */
-public class RemoteSite {
-
-    /* our log stream */
-    private final static Logger LOG = Logger.getLogger(RemoteSite.class
-        .getName());
-
-    private String alias, username, password, cdTestDir;
-    private int maxConnections;
-    private URL url;
-
-    public RemoteSite(String alias, URL url, String username, String password) {
-        this.alias = alias;
-        this.username = username;
-        this.password = password;
-        this.url = url;
-        this.maxConnections = -1;
-    }
-    
-    public RemoteSite(String alias, URL url, String username, String password, String cdTestDir) {
-        this(alias, url, username, password);
-        this.cdTestDir = cdTestDir;
-    }
-    
-    public RemoteSite(String alias, URL url, String username, String password, String cdTestDir, int maxConnections) {
-        this(alias, url, username, password, cdTestDir);
-        this.maxConnections = maxConnections;
-    }
-
-    public String getAlias() {
-        return this.alias;
-    }
-
-    public URL getURL() {
-        return this.url;
-    }
-
-    public String getUsername() {
-        return this.username;
-    }
-
-    public String getPassword() {
-        return this.password;
-    }
-    
-    public String getCdTestDir() {
-        return this.cdTestDir;
-    }
-
-    public int getMaxConnections() {
-    	return this.maxConnections;
-    }
-    
-    public void copy(RemoteSite rs) {
-        this.alias = rs.alias;
-        this.url = rs.url;
-        this.username = rs.username;
-        this.password = rs.password;
-        this.cdTestDir = rs.cdTestDir;
-        this.maxConnections = rs.maxConnections;
-    }
-
-    public boolean equals(Object obj) {
-        if (obj instanceof RemoteSite) {
-            RemoteSite rs = (RemoteSite) obj;
-            try {
-                return (rs.alias.equals(this.alias) && rs.url.toURI().equals(this.url.toURI())
-                        && rs.username.equals(this.username) && rs.password
-                        .equals(this.password) && rs.maxConnections == this.maxConnections);
-            } catch (URISyntaxException e) {
-                LOG.log(Level.SEVERE, "Could not convert URL to URL: Message: "+e.getMessage());
-            }
-        } else {
-            return false;
-        }
-        return false;
-    }
-
-    public String toString() {
-        return "RemoteSite: alias = '" + this.alias + "'  url = '" + this.url
-                + "'  username = '" + this.username + "' cdTestDir = '" 
-                + this.cdTestDir + "' maxConnections = '" + this.maxConnections + "'";
-    }
-
-    @Override
-    public int hashCode() {
-        int result = alias != null ? alias.hashCode() : 0;
-        result = 31 * result + (username != null ? username.hashCode() : 0);
-        result = 31 * result + (password != null ? password.hashCode() : 0);
-        result = 31 * result + (cdTestDir != null ? cdTestDir.hashCode() : 0);
-        result = 31 * result + maxConnections;
-        try {
-            result = 31 * result + (url != null ? url.toURI().hashCode() : 0);
-        } catch (URISyntaxException e) {
-            e.printStackTrace();
-        }
-        return result;
-    }
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
deleted file mode 100644
index 89bfdbd..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.cas.pushpull.protocol;
-
-//OODT imports
-import org.apache.oodt.cas.protocol.ProtocolFile;
-
-/**
- *
- * Extends {@link ProtocolFile} and links it to a {@link RemoteSite}.
- *
- * @author mattmann
- * @version $Revision$
- *
- */
-public class RemoteSiteFile extends ProtocolFile {
-
-  private RemoteSite site;
-
-  public RemoteSiteFile(ProtocolFile file, RemoteSite site){
-     this(file.getPath(), file.isDir(), site);
-  }
-
-  /**
-   * @param parent
-   * @param path
-   * @param isDir
-   * @param site
-   */
-  public RemoteSiteFile(ProtocolFile parent, String path, boolean isDir,
-      RemoteSite site) {
-    super(parent, path, isDir);
-    this.site = site;
-  }
-
-  /**
-   * @param path
-   * @param isDir
-   * @param site
-   */
-  public RemoteSiteFile(String path, boolean isDir, RemoteSite site) {
-    super(path, isDir);
-    this.site = site;
-  }
-
-  /**
-   * @return the site
-   */
-  public RemoteSite getSite() {
-    return site;
-  }
-
-  /**
-   * @param site
-   *          the site to set
-   */
-  public void setSite(RemoteSite site) {
-    this.site = site;
-  }
-
-  public RemoteSiteFile getRemoteParent() {
-    ProtocolFile parent = super.getParent();
-    return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
-  }
-
-  @Override
-public RemoteSiteFile getAbsoluteFile() {
-    ProtocolFile parent = super.getAbsoluteFile();
-    return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/ListRetriever.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/ListRetriever.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/ListRetriever.java
deleted file mode 100644
index dbfa94f..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/ListRetriever.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.retrievalmethod;
-
-//OODT imports
-import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
-import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.pushpull.config.DataFilesInfo;
-import org.apache.oodt.cas.pushpull.config.DownloadInfo;
-import org.apache.oodt.cas.pushpull.exceptions.AlreadyInDatabaseException;
-import org.apache.oodt.cas.pushpull.exceptions.ParserException;
-import org.apache.oodt.cas.pushpull.exceptions.RetrievalMethodException;
-import org.apache.oodt.cas.pushpull.exceptions.ToManyFailedDownloadsException;
-import org.apache.oodt.cas.pushpull.exceptions.UndefinedTypeException;
-import org.apache.oodt.cas.pushpull.filerestrictions.FileRestrictions;
-import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFileStructure;
-import org.apache.oodt.cas.pushpull.protocol.RemoteSite;
-import org.apache.oodt.cas.pushpull.retrievalsystem.DataFileToPropFileLinker;
-import org.apache.oodt.cas.pushpull.retrievalsystem.FileRetrievalSystem;
-
-
-//JDK imports
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.util.LinkedList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- * 
- * <p>
- * Describe your class here
- * </p>.
- */
-public class ListRetriever implements RetrievalMethod {
-
-    /* our log stream */
-    private static final Logger LOG = Logger.getLogger(ListRetriever.class
-            .getName());
-
-    public void processPropFile(FileRetrievalSystem frs, Parser propFileParser,
-            File propFile, DataFilesInfo dfi, DataFileToPropFileLinker linker)
-        throws FileNotFoundException, ParserException, RetrievalMethodException {
-        RemoteSite remoteSite;
-
-        // parse property file
-        Metadata fileMetadata = new Metadata();
-        VirtualFileStructure vfs = propFileParser.parse(new FileInputStream(
-                propFile), fileMetadata);
-        DownloadInfo di = dfi.getDownloadInfo();
-        if (!di.isAllowAliasOverride()
-                || (remoteSite = vfs.getRemoteSite()) == null) {
-          remoteSite = di.getRemoteSite();
-        }
-        LinkedList<String> fileList = FileRestrictions.toStringList(vfs
-                .getRootVirtualFile());
-
-        // download data files specified in property file
-        for (String file : fileList) {
-            try {
-                linker.addPropFileToDataFileLink(propFile, file);
-                if (!frs.addToDownloadQueue(remoteSite, file, di
-                        .getRenamingConv(), di.getStagingArea(), dfi
-                        .getQueryMetadataElementName(), di.deleteFromServer(), fileMetadata)) {
-                  linker.eraseLinks(propFile);
-                }
-            } catch (ToManyFailedDownloadsException e) {
-                throw new RetrievalMethodException(
-                        "Connection appears to be down. . .unusual number of download failures. . .stopping : "
-                                + e.getMessage());
-            } catch (CatalogException e) {
-                throw new RetrievalMethodException(
-                        "Failed to communicate with database : "
-                                + e.getMessage());
-            } catch (AlreadyInDatabaseException e) {
-                LOG.log(Level.WARNING, "Skipping file : " + e.getMessage());
-            } catch (UndefinedTypeException e) {
-                LOG.log(Level.WARNING, "Skipping file : " + e.getMessage());
-            } catch (Exception e) {
-                LOG.log(Level.SEVERE, e.getMessage());
-                linker.markAsFailed(propFile, "Failed to download " + file
-                        + " from " + remoteSite + " : " + e.getMessage());
-                throw new RetrievalMethodException("Uknown error accured while downloading "
-                        + file + " from " + remoteSite + " -- bailing out : "
-                        + e.getMessage());
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
deleted file mode 100644
index d3030e3..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.retrievalmethod;
-
-//OODT imports
-import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
-import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.protocol.exceptions.ProtocolException;
-import org.apache.oodt.cas.pushpull.config.DataFilesInfo;
-import org.apache.oodt.cas.pushpull.config.DownloadInfo;
-import org.apache.oodt.cas.pushpull.exceptions.AlreadyInDatabaseException;
-import org.apache.oodt.cas.pushpull.exceptions.ParserException;
-import org.apache.oodt.cas.pushpull.exceptions.RetrievalMethodException;
-import org.apache.oodt.cas.pushpull.exceptions.ToManyFailedDownloadsException;
-import org.apache.oodt.cas.pushpull.exceptions.UndefinedTypeException;
-import org.apache.oodt.cas.pushpull.filerestrictions.FileRestrictions;
-import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFile;
-import org.apache.oodt.cas.pushpull.filerestrictions.VirtualFileStructure;
-import org.apache.oodt.cas.protocol.ProtocolFile;
-import org.apache.oodt.cas.protocol.util.ProtocolFileFilter;
-import org.apache.oodt.cas.pushpull.protocol.ProtocolPath;
-import org.apache.oodt.cas.pushpull.protocol.RemoteSite;
-import org.apache.oodt.cas.pushpull.protocol.RemoteSiteFile;
-import org.apache.oodt.cas.pushpull.retrievalsystem.DataFileToPropFileLinker;
-import org.apache.oodt.cas.pushpull.retrievalsystem.FileRetrievalSystem;
-
-
-//JDK imports
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.net.MalformedURLException;
-import java.util.List;
-import java.util.Stack;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- *
- * @author bfoster
- * @version $Revision$
- *
- * <p>
- * Describe your class here
- * </p>.
- */
-public class RemoteCrawler implements RetrievalMethod {
-
-    private static final Logger LOG = Logger.getLogger(RemoteCrawler.class
-            .getName());
-
-    /**
-     * Starts the crawler and creates a default DirStruct if null was supplied
-     * in constructor
-     *
-     * @throws java.net.MalformedURLException
-     * @throws org.apache.oodt.cas.pushpull.exceptions.ProtocolException
-     * @throws org.apache.oodt.cas.pushpull.exceptions.ProtocolFileException
-     */
-    @Override
-   public void processPropFile(FileRetrievalSystem frs, Parser propFileParser,
-            File propFile, DataFilesInfo dfi, DataFileToPropFileLinker linker)
-        throws FileNotFoundException, ParserException, ProtocolException, MalformedURLException,
-        RetrievalMethodException {
-        RemoteSite remoteSite;
-
-        // parse property file
-        Metadata fileMetadata = new Metadata();
-        VirtualFileStructure vfs = propFileParser.parse(new FileInputStream(
-                propFile), fileMetadata);
-
-        // determine RemoteSite
-        DownloadInfo di = dfi.getDownloadInfo();
-        if (!di.isAllowAliasOverride()
-                || (remoteSite = vfs.getRemoteSite()) == null) {
-          remoteSite = di.getRemoteSite();
-        }
-
-        // modify vfs to be root based if HOME directory based
-        if (!vfs.isRootBased()) {
-            String homeDirPath = frs.getHomeDir(remoteSite).getPath();
-            VirtualFile root = new VirtualFile(homeDirPath, true);
-            root.addChild(vfs.getRootVirtualFile());
-            vfs = new VirtualFileStructure(homeDirPath + "/"
-                    + vfs.getPathToRoot(), root.getRootDir());
-            frs.changeToHOME(remoteSite);
-        }
-
-        // initialize variables
-        final String initialCdPath = vfs.getPathToRoot();
-        final VirtualFile vf = vfs.getRootVirtualFile();
-
-        // change to initial directory (takes care of Linux auto-mounting)
-        frs.changeToDir(initialCdPath, remoteSite);
-
-        // add starting directory to stack
-        Stack<RemoteSiteFile> files = new Stack<RemoteSiteFile>();
-        files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite), remoteSite));
-
-        // start crawling
-        while (!files.isEmpty()) {
-            RemoteSiteFile file = files.peek();
-            try {
-                // if directory, then add its children to the crawl list
-                if (file.isDir()) {
-
-                    // get next page worth of children
-                    List<RemoteSiteFile> children = frs.getNextPage(file,
-                            new ProtocolFileFilter() {
-                                @Override
-                              public boolean accept(ProtocolFile pFile) {
-                                    return FileRestrictions.isAllowed(new
-                                        ProtocolPath(pFile
-                                            .getPath(), pFile.isDir()), vf);
-                                }
-                            });
-
-                    // if directory had more children then add them
-                    if (children.size() > 0) {
-                      files.addAll(children);
-                    }// otherwise remove the directory from the crawl list
-                    else {
-                      files.pop();
-                    }
-
-                    // if file, then download it
-                } else {
-                    linker.addPropFileToDataFileLink(propFile, file);
-                    if (!frs.addToDownloadQueue(files.pop(), di
-                            .getRenamingConv(), di.getStagingArea(), dfi
-                            .getQueryMetadataElementName(), di
-                            .deleteFromServer(), fileMetadata)) {
-                      linker.eraseLinks(propFile);
-                    }
-                }
-
-            } catch (ToManyFailedDownloadsException e) {
-                throw new RetrievalMethodException(
-                        "Connection appears to be down. . .unusual number of download failures. . .stopping : "
-                                + e.getMessage());
-            } catch (CatalogException e) {
-                throw new RetrievalMethodException(
-                        "Failed to communicate with database : "
-                                + e.getMessage());
-            } catch (AlreadyInDatabaseException e) {
-                LOG.log(Level.WARNING, "Skipping file : " + e.getMessage());
-            } catch (UndefinedTypeException e) {
-                LOG.log(Level.WARNING, "Skipping file : " + e.getMessage());
-            } catch (Exception e) {
-                linker.markAsFailed(propFile, e.getMessage());
-                throw new RetrievalMethodException("Uknown error accured while downloading "
-                        + file + " from " + remoteSite + " -- bailing out : "
-                        + e.getMessage(), e);
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/oodt/blob/098cc4fa/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RetrievalMethod.java
----------------------------------------------------------------------
diff --git a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RetrievalMethod.java b/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RetrievalMethod.java
deleted file mode 100644
index d34874b..0000000
--- a/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RetrievalMethod.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oodt.cas.pushpull.retrievalmethod;
-
-//JDK imports
-import java.io.File;
-
-//OODT imports
-import org.apache.oodt.cas.pushpull.config.DataFilesInfo;
-import org.apache.oodt.cas.pushpull.filerestrictions.Parser;
-import org.apache.oodt.cas.pushpull.retrievalsystem.DataFileToPropFileLinker;
-import org.apache.oodt.cas.pushpull.retrievalsystem.FileRetrievalSystem;
-
-/**
- * 
- * @author bfoster
- * @version $Revision$
- * 
- * <p>
- * Describe your class here
- * </p>.
- */
-public interface RetrievalMethod {
-
-    void processPropFile(FileRetrievalSystem frs,
-                         Parser propFileParser, File propFile, DataFilesInfo dfi,
-                         DataFileToPropFileLinker linker) throws Exception;
-
-}


Mime
View raw message