manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From min...@apache.org
Subject svn commit: r1495913 [2/5] - in /manifoldcf/branches/CONNECTORS-728: ./ connectors/ connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/ connectors/filesystem/connector/src/main/native2ascii/org/apache/mani...
Date Mon, 24 Jun 2013 03:26:58 GMT
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,153 @@
+/* $Id: FileOutputSpecs.java 1299512 2013-05-31 22:58:38Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class HDFSOutputSpecs extends HDFSOutputParam {
+  /**
+   * 
+   */
+  private static final long serialVersionUID = 1145652730572662025L;
+
+  final public static ParameterEnum[] SPECIFICATIONLIST = {
+    ParameterEnum.ROOTPATH
+  };
+
+  private String rootPath;
+
+  /** Build a set of ElasticSearch parameters by reading an JSON object
+   * 
+   * @param json
+   * @throws JSONException
+   * @throws ManifoldCFException
+   */
+  public HDFSOutputSpecs(String json) throws JSONException, ManifoldCFException {
+    this(new JSONObject(json));
+  }
+
+  /** Build a set of ElasticSearch parameters by reading an JSON object
+   * 
+   * @param json
+   * @throws JSONException
+   * @throws ManifoldCFException
+   */
+  public HDFSOutputSpecs(JSONObject json) throws JSONException, ManifoldCFException {
+    super(SPECIFICATIONLIST);
+    rootPath = null;
+    for (ParameterEnum param : SPECIFICATIONLIST) {
+      String value = null;
+      value = json.getString(param.name());
+      if (value == null) {
+        value = param.defaultValue;
+      }
+      put(param, value);
+    }
+    rootPath = getRootPath();
+  }
+
+  /** Build a set of ElasticSearch parameters by reading an instance of
+   * SpecificationNode.
+   * 
+   * @param node
+   * @throws ManifoldCFException
+   */
+  public HDFSOutputSpecs(ConfigurationNode node) throws ManifoldCFException {
+    super(SPECIFICATIONLIST);
+    rootPath = null;
+    for (ParameterEnum param : SPECIFICATIONLIST) {
+      String value = null;
+      if (node != null) {
+        value = node.getAttributeValue(param.name());
+      }
+      if (value == null) {
+        value = param.defaultValue;
+      }
+      put(param, value);
+    }
+    rootPath = getRootPath();
+  }
+
+  /**
+   * @param variableContext
+   * @param specNode
+   */
+  public static void contextToSpecNode(IPostParameters variableContext, ConfigurationNode specNode) {
+    for (ParameterEnum param : SPECIFICATIONLIST) {
+      String p = variableContext.getParameter(param.name().toLowerCase());
+      if (p != null) {
+        specNode.setAttribute(param.name(), p);
+      }
+    }
+  }
+
+  /** @return a JSON representation of the parameter list */
+  public JSONObject toJson() {
+    return new JSONObject(this);
+  }
+
+  /**
+   * @return
+   */
+  public String getRootPath() {
+    return get(ParameterEnum.ROOTPATH);
+  }
+
+  /**
+   * @param content
+   * @return
+   * @throws ManifoldCFException
+   */
+  private final static TreeSet<String> createStringSet(String content) throws ManifoldCFException {
+    TreeSet<String> set = new TreeSet<String>();
+    BufferedReader br = null;
+    StringReader sr = null;
+    try {
+      sr = new StringReader(content);
+      br = new BufferedReader(sr);
+      String line = null;
+      while ((line = br.readLine()) != null) {
+        line = line.trim();
+        if (line.length() > 0) {
+          set.add(line);
+        }
+      }
+      return set;
+    } catch (IOException e) {
+      throw new ManifoldCFException(e);
+    } finally {
+      if (br != null) {
+        IOUtils.closeQuietly(br);
+      }
+    }
+  }
+
+}

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,141 @@
+/* $Id: Messages.java 1295926 2013-05-31 23:00:00Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+  public static final String DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.agents.output.hdfs.common";
+  public static final String DEFAULT_PATH_NAME="org.apache.manifoldcf.agents.output.hdfs";
+
+  /** Constructor - do no instantiate
+   */
+  protected Messages()
+  {
+  }
+
+  public static String getString(Locale locale, String messageKey)
+  {
+    return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getAttributeString(Locale locale, String messageKey)
+  {
+    return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getBodyString(Locale locale, String messageKey)
+  {
+    return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getAttributeJavascriptString(Locale locale, String messageKey)
+  {
+    return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getBodyJavascriptString(Locale locale, String messageKey)
+  {
+    return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getString(Locale locale, String messageKey, Object[] args)
+  {
+    return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getAttributeString(Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getBodyString(Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getAttributeJavascriptString(Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getBodyJavascriptString(Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  // More general methods which allow bundlenames and class loaders to be specified.
+
+  public static String getString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getAttributeString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getBodyString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getAttributeJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getBodyJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  // Resource output
+
+  public static void outputResource(IHTTPOutput output, Locale locale, String resourceKey,
+      Map<String,String> substitutionParameters, boolean mapToUpperCase)
+          throws ManifoldCFException
+          {
+    outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+        substitutionParameters,mapToUpperCase);
+          }
+
+  public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+      Map<String,String> substitutionParameters, boolean mapToUpperCase)
+          throws ManifoldCFException
+          {
+    outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+        substitutionParameters,mapToUpperCase);
+          }
+
+  public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+      Map<String,Object> contextObjects)
+          throws ManifoldCFException
+          {
+    outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+        contextObjects);
+          }
+
+}
+

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,1503 @@
+/* $Id: FileConnector.java 995085 2010-09-08 15:13:38Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.Logging;
+import org.apache.manifoldcf.core.extmimemap.ExtensionMimeMap;
+import java.util.*;
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/** This is the "repository connector" for a file system.  It's a relative of the share crawler, and should have
+* comparable basic functionality, with the exception of the ability to use ActiveDirectory and look at other shares.
+*/
+public class HDFSRepositoryConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
+{
+  public static final String _rcsid = "@(#)$Id: FileConnector.java 995085 2010-09-08 15:13:38Z kwright $";
+
+  // Activities that we know about
+  protected final static String ACTIVITY_READ = "read document";
+
+  // Relationships we know about
+  protected static final String RELATIONSHIP_CHILD = "child";
+
+  // Activities list
+  protected static final String[] activitiesList = new String[]{ACTIVITY_READ};
+
+  // Parameters that this connector cares about
+  // public final static String ROOTDIRECTORY = "rootdirectory";
+
+  // Local data
+  // protected File rootDirectory = null;
+
+  protected Configuration config = null;
+  protected FileSystem fileSystem = null;
+
+  /*
+   * Constructor.
+   */
+  public HDFSRepositoryConnector()
+  {
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.manifoldcf.core.connector.BaseConnector#connect(org.apache.manifoldcf.core.interfaces.ConfigParams)
+   */
+  @Override
+  public void connect(ConfigParams configParams) {
+    super.connect(configParams);
+
+    String nameNode = configParams.getParameter("namenode");
+    
+    String user = configParams.getParameter("user");
+    
+    /*
+     * make Configuration
+     */
+    ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+    try {
+      Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
+      config = new Configuration();
+      config.set("fs.default.name", nameNode);
+    } finally {
+      Thread.currentThread().setContextClassLoader(ocl);
+    }
+    
+    /*
+     * get connection to HDFS
+     */
+    try {
+      fileSystem = FileSystem.get(new URI(nameNode), config, user);
+	} catch (URISyntaxException e) {
+      e.printStackTrace();
+	} catch (IOException e) {
+      e.printStackTrace();
+	} catch (InterruptedException e) {
+      e.printStackTrace();
+	}
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.manifoldcf.core.connector.BaseConnector#disconnect()
+   */
+  @Override
+  public void disconnect() throws ManifoldCFException {
+    try {
+      fileSystem.close();
+    } catch(IOException ex) {
+      throw new ManifoldCFException(ex);
+    }
+    config.clear();
+    super.disconnect();
+  }
+
+  /** Tell the world what model this connector uses for getDocumentIdentifiers().
+   * This must return a model value as specified above.
+   *@return the model type value.
+   */
+  @Override
+  public int getConnectorModel()
+  {
+    return MODEL_CHAINED_ADD_CHANGE;
+  }
+
+  /** Return the list of relationship types that this connector recognizes.
+   *@return the list.
+   */
+  @Override
+  public String[] getRelationshipTypes()
+  {
+    return new String[]{RELATIONSHIP_CHILD};
+  }
+
+  /** List the activities we might report on.
+   */
+  @Override
+  public String[] getActivitiesList()
+  {
+    return activitiesList;
+  }
+
+  /** For any given document, list the bins that it is a member of.
+   */
+  @Override
+  public String[] getBinNames(String documentIdentifier)
+  {
+    /*
+    // Note: This code is for testing, so we can see how documents behave when they are in various kinds of bin situations.
+    // The testing model is that there are documents belonging to "SLOW", to "FAST", or both to "SLOW" and "FAST" bins.
+    // The connector chooses which bins to assign a document to based on the identifier (which is the document's path), so
+    // this is something that should NOT be duplicated by other connector implementers.
+    if (documentIdentifier.indexOf("/BOTH/") != -1 || (documentIdentifier.indexOf("/SLOW/") != -1 && documentIdentifier.indexOf("/FAST/") != -1))
+      return new String[]{"SLOW","FAST"};
+    if (documentIdentifier.indexOf("/SLOW/") != -1)
+      return new String[]{"SLOW"};
+    if (documentIdentifier.indexOf("/FAST/") != -1)
+      return new String[]{"FAST"};
+     */
+    return new String[]{""};
+  }
+
+  /** Convert a document identifier to a URI.  The URI is the URI that will be the unique key from
+  * the search index, and will be presented to the user as part of the search results.
+  *@param filePath is the document filePath.
+  *@param repositoryPath is the document repositoryPath.
+  *@return the document uri.
+  */
+  protected String convertToURI(String documentIdentifier, String[] repositoryPaths)
+    throws ManifoldCFException
+  {
+    //
+    // Note well:  This MUST be a legal URI!!!
+    try
+    {
+      String path = new Path(documentIdentifier).toString();
+      for (String repositoryPath : repositoryPaths) {
+        if (path.startsWith(repositoryPath)) {
+          StringBuffer sb = new StringBuffer();
+          path = path.replaceFirst(repositoryPath, "");
+          if (path.startsWith("/")) {
+            path = path.replaceFirst("/", "");
+          }
+          String[] tmp = path.split("/", 3);
+          String scheme = "";
+          String host = "";
+          String other = "";
+          try {
+            scheme = tmp[0];
+          } catch (ArrayIndexOutOfBoundsException e) {
+            scheme = "hdfs";
+          }
+          try {
+            host = tmp[1];
+          } catch (ArrayIndexOutOfBoundsException e) {
+            host = "localhost:9000";
+          }
+          try {
+            other = "/" + tmp[2];
+          } catch (ArrayIndexOutOfBoundsException e) {
+            other = "/";
+          }
+          return new URI(scheme + "://" + host + other).toURL().toString();
+        }
+      }
+      return convertToURI(documentIdentifier);
+    }
+    catch (URISyntaxException e)
+    {
+      throw new ManifoldCFException("Bad url",e);
+    }
+    catch (IOException e)
+    {
+      throw new ManifoldCFException("Bad url",e);
+    }
+  }
+  
+  /** Convert a document identifier to a URI.  The URI is the URI that will be the unique key from
+  * the search index, and will be presented to the user as part of the search results.
+  *@param documentIdentifier is the document identifier.
+  *@return the document uri.
+  */
+  protected String convertToURI(String documentIdentifier)
+    throws ManifoldCFException
+  {
+    //
+    // Note well:  This MUST be a legal URI!!!
+    return new Path(documentIdentifier).toUri().toString();
+  }
+
+
+  /** Given a document specification, get either a list of starting document identifiers (seeds),
+  * or a list of changes (deltas), depending on whether this is a "crawled" connector or not.
+  * These document identifiers will be loaded into the job's queue at the beginning of the
+  * job's execution.
+  * This method can return changes only (because it is provided a time range).  For full
+  * recrawls, the start time is always zero.
+  * Note that it is always ok to return MORE documents rather than less with this method.
+  *@param spec is a document specification (that comes from the job).
+  *@param startTime is the beginning of the time range to consider, inclusive.
+  *@param endTime is the end of the time range to consider, exclusive.
+  *@return the stream of local document identifiers that should be added to the queue.
+  */
+  @Override
+  public IDocumentIdentifierStream getDocumentIdentifiers(DocumentSpecification spec, long startTime, long endTime)
+    throws ManifoldCFException
+  {
+    return new IdentifierStream(spec);
+  }
+
+
+  /** Get document versions given an array of document identifiers.
+  * This method is called for EVERY document that is considered. It is therefore important to perform
+  * as little work as possible here.
+  * The connector will be connected before this method can be called.
+  *@param documentIdentifiers is the array of local document identifiers, as understood by this connector.
+  *@param oldVersions is the corresponding array of version strings that have been saved for the document identifiers.
+  *   A null value indicates that this is a first-time fetch, while an empty string indicates that the previous document
+  *   had an empty version string.
+  *@param activities is the interface this method should use to perform whatever framework actions are desired.
+  *@param spec is the current document specification for the current job.  If there is a dependency on this
+  * specification, then the version string should include the pertinent data, so that reingestion will occur
+  * when the specification changes.  This is primarily useful for metadata.
+  *@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
+  *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
+  *@return the corresponding version strings, with null in the places where the document no longer exists.
+  * Empty version strings indicate that there is no versioning ability for the corresponding document, and the document
+  * will always be processed.
+  */
+  public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
+    DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    int i = 0;
+    
+    /*
+     * get filepathtouri value
+     */
+    boolean filePathToUri = false;
+    i = 0;
+    while (i < spec.getChildCount()) {
+      SpecificationNode sn = spec.getChild(i++);
+      if (sn.getType().equals("filepathtouri")) {
+        filePathToUri = Boolean.valueOf(sn.getValue());
+      }
+    }
+
+    String[] rval = new String[documentIdentifiers.length];
+    try
+    {
+      i = 0;
+      while (i < rval.length)
+      {
+        Path path = new Path(documentIdentifiers[i]);
+        if (fileSystem.exists(path))
+        {
+          if (fileSystem.getFileStatus(path).isDir())
+          {
+            // It's a directory.  The version ID will be the
+            // last modified date.
+            long lastModified = fileSystem.getFileStatus(path).getModificationTime();
+            rval[i] = new Long(lastModified).toString();
+
+            // Signal that we don't have any versioning.
+            // rval[i] = "";
+          }
+          else
+          {
+            // It's a file
+            long fileLength = fileSystem.getFileStatus(path).getLen();
+            if (activities.checkLengthIndexable(fileLength))
+            {
+              // Get the file's modified date.
+              long lastModified = fileSystem.getFileStatus(path).getModificationTime();
+              StringBuilder sb = new StringBuilder();
+              if (filePathToUri)
+              {
+                sb.append("+");
+              }
+              else
+              {
+                sb.append("-");
+              }
+              sb.append(new Long(lastModified).toString()).append(":").append(new Long(fileLength).toString());
+              rval[i] = sb.toString();
+            }
+            else
+            {
+              rval[i] = null;
+            }
+          }
+        }
+        else
+        {
+          rval[i] = null;
+        }
+        i++;
+      }
+    }
+    catch (IOException e)
+    {
+      throw new ManifoldCFException(e);
+    }
+    return rval;
+  }
+
+
+  /** Process a set of documents.
+  * This is the method that should cause each document to be fetched, processed, and the results either added
+  * to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
+  * The document specification allows this class to filter what is done based on the job.
+  *@param documentIdentifiers is the set of document identifiers to process.
+  *@param activities is the interface this method should use to queue up new document references
+  * and ingest documents.
+  *@param spec is the document specification.
+  *@param scanOnly is an array corresponding to the document identifiers.  It is set to true to indicate when the processing
+  * should only find other references, and should not actually call the ingestion methods.
+  */
+  @Override
+  public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities, DocumentSpecification spec, boolean[] scanOnly)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    try
+    {
+      int i = 0;
+      while (i < documentIdentifiers.length)
+      {
+        String version = versions[i];
+        String documentIdentifier = documentIdentifiers[i];
+        Path path = new Path(documentIdentifier);
+        FileStatus fileStatus = fileSystem.getFileStatus(path);
+        if (fileSystem.exists(path))
+        {
+          if (fileStatus.isDir())
+          {
+            // Queue up stuff for directory
+            long startTime = System.currentTimeMillis();
+            String errorCode = "OK";
+            String errorDesc = null;
+            String entityReference = documentIdentifier;
+            try
+            {
+              try
+              {
+                FileStatus[] fileStatuses = fileSystem.listStatus(path);
+                if (fileStatuses != null)
+                {
+                  int j = 0;
+                  while (j < fileStatuses.length)
+                  {
+                    FileStatus fs = fileStatuses[j++];
+                    String canonicalPath = fs.getPath().toString();
+                    if (checkInclude(fileSystem.getUri().toString(),fs,canonicalPath,spec))
+                      activities.addDocumentReference(canonicalPath,documentIdentifier,RELATIONSHIP_CHILD);
+                  }
+                }
+              }
+              catch (IOException e)
+              {
+                errorCode = "IO ERROR";
+                errorDesc = e.getMessage();
+                throw new ManifoldCFException("IO Error: "+e.getMessage(),e);
+              }
+            }
+            finally
+            {
+              activities.recordActivity(new Long(startTime),ACTIVITY_READ,null,entityReference,errorCode,errorDesc,null);
+            }
+          }
+          else
+          {
+            if (!scanOnly[i])
+            {
+              // We've already avoided queuing documents that we don't want, based on file specifications.
+              // We still need to check based on file data.
+              if (checkIngest(fileSystem.getUri().toString(),fileStatus,spec))
+              {
+                int j = 0;
+
+                /*
+                 * get repository paths
+                 */
+                j = 0;
+                List<String> repositoryPaths = new ArrayList<String>();
+                while ( j < spec.getChildCount())
+                {
+                  SpecificationNode sn = spec.getChild(j++);
+                  if (sn.getType().equals("startpoint"))
+                  {
+                    if (sn.getAttributeValue("path").length() > 0) {
+                      repositoryPaths.add(fileSystem.getUri().resolve(sn.getAttributeValue("path")).toString());
+                    }
+                  }
+                }
+
+                /*
+                 * get filepathtouri value
+                 */
+                boolean filePathToUri = false;
+                if (version.length() > 0 && version.startsWith("+")) {
+                  filePathToUri = true;
+                }
+
+                long startTime = System.currentTimeMillis();
+                String errorCode = "OK";
+                String errorDesc = null;
+                Long fileLength = null;
+                String entityDescription = documentIdentifier;
+                try
+                {
+                  // Ingest the document.
+                  try
+                  {
+                    FSDataInputStream is = fileSystem.open(path);
+                    try
+                    {
+                      long fileBytes = fileStatus.getLen();
+                      RepositoryDocument data = new RepositoryDocument();
+                      data.setBinary(is,fileBytes);
+                      String fileName = path.getName();
+                      data.setFileName(fileName);
+                      data.setMimeType(mapExtensionToMimeType(fileName));
+                      data.setModifiedDate(new Date(fileStatus.getModificationTime()));
+                      if (filePathToUri) {
+                        data.addField("uri",convertToURI(documentIdentifier,repositoryPaths.toArray(new String[0])));
+                        // MHL for other metadata
+                        activities.ingestDocument(documentIdentifier,version,convertToURI(documentIdentifier,repositoryPaths.toArray(new String[0])),data);
+                      } else {
+                        data.addField("uri",path.toString());
+                        // MHL for other metadata
+                        activities.ingestDocument(documentIdentifier,version,convertToURI(documentIdentifier),data);
+                      }
+                      fileLength = new Long(fileBytes);
+                    }
+                    finally
+                    {
+                      is.close();
+                    }
+                  }
+                  catch (IOException e)
+                  {
+                    errorCode = "IO ERROR";
+                    errorDesc = e.getMessage();
+                    throw new ManifoldCFException("IO Error: "+e.getMessage(),e);
+                  }
+                }
+                finally
+                {
+                  activities.recordActivity(new Long(startTime),ACTIVITY_READ,fileLength,entityDescription,errorCode,errorDesc,null);
+                }
+              }
+            }
+          }
+        }
+        i++;
+      }
+    }
+    catch(IOException e)
+    {
+      throw new ManifoldCFException(e);
+    }
+  }
+
+  /** Map an extension to a mime type */
+  protected static String mapExtensionToMimeType(String fileName)
+  {
+    int slashIndex = fileName.lastIndexOf("/");
+    if (slashIndex != -1)
+      fileName = fileName.substring(slashIndex+1);
+    int dotIndex = fileName.lastIndexOf(".");
+    if (dotIndex == -1)
+      return null;
+    return ExtensionMimeMap.mapToMimeType(fileName.substring(dotIndex+1).toLowerCase(java.util.Locale.ROOT));
+  }
+
+  // UI support methods.
+  //
+  // These support methods come in two varieties.  The first bunch is involved in setting up connection configuration information.  The second bunch
+  // is involved in presenting and editing document specification information for a job.  The two kinds of methods are accordingly treated differently,
+  // in that the first bunch cannot assume that the current connector object is connected, while the second bunch can.  That is why the first bunch
+  // receives a thread context argument for all UI methods, while the second bunch does not need one (since it has already been applied via the connect()
+  // method, above).
+    
+  /** Output the configuration header section.
+  * This method is called in the head section of the connector's configuration page.  Its purpose is to add the required tabs to the list, and to output any
+  * javascript methods that might be needed by the configuration editing HTML.
+  *@param threadContext is the local thread context.
+  *@param out is the output to which any HTML should be sent.
+  *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+  *@param tabsArray is an array of tab names.  Add to this array any tab names that are specific to the connector.
+  */
+  @Override
+  public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray) throws ManifoldCFException, IOException
+  {
+    tabsArray.add(Messages.getString(locale,"HDFSRepositoryConnector.ServerTabName"));
+    
+    out.print(
+"<script type=\"text/javascript\">\n"+
+"<!--\n"+
+"function checkConfigForSave()\n"+
+"{\n"+
+"  return true;\n"+
+"}\n"+
+"\n"+
+"//-->\n"+
+"</script>\n"
+    );
+  }
+  
+  /** Output the configuration body section.
+  * This method is called in the body section of the connector's configuration page.  Its purpose is to present the required form elements for editing.
+  * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags.  The name of the
+  * form is "editconnection".
+  *@param threadContext is the local thread context.
+  *@param out is the output to which any HTML should be sent.
+  *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+  *@param tabName is the current tab name.
+  */
+  public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, String tabName)
+    throws ManifoldCFException, IOException
+  {
+    String nameNode = parameters.getParameter("namenode");
+    if (nameNode == null) {
+    	nameNode = "hdfs://localhost:9000";
+    }
+    String user = parameters.getParameter("user");
+    if (user == null) {
+    	user = "";
+    }
+    
+    if (tabName.equals(Messages.getString(locale,"HDFSRepositoryConnector.ServerTabName")))
+    {
+      out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NameNode") + "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"namenode\" type=\"text\" size=\"48\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nameNode)+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.User") + "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"user\" type=\"text\" size=\"48\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(user)+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
+"</table>\n"
+      );
+    }
+    else
+    {
+      // Server tab hiddens
+      out.print(
+"<input type=\"hidden\" name=\"namenode\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nameNode)+"\"/>\n"+
+"<input type=\"hidden\" name=\"user\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(user)+"\"/>\n"
+      );
+    }
+  }
+  
+  /** Process a configuration post.
+  * This method is called at the start of the connector's configuration page, whenever there is a possibility that form data for a connection has been
+  * posted.  Its purpose is to gather form information and modify the configuration parameters accordingly.
+  * The name of the posted form is "editconnection".
+  *@param threadContext is the local thread context.
+  *@param variableContext is the set of variables available from the post, including binary file post information.
+  *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+  *@return null if all is well, or a string error message if there is an error that should prevent saving of the connection (and cause a redirection to an error page).
+  */
+  @Override
+  public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext, ConfigParams parameters)
+    throws ManifoldCFException
+  {
+    String nameNode = variableContext.getParameter("namenode");
+    if (nameNode != null) {
+      parameters.setParameter("namenode", nameNode);
+    }
+
+    String user = variableContext.getParameter("user");
+    if (user != null) {
+      parameters.setParameter("user", user);
+    }
+
+    return null;
+  }
+  
+  /** View configuration.
+  * This method is called in the body section of the connector's view configuration page.  Its purpose is to present the connection information to the user.
+  * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+  *@param threadContext is the local thread context.
+  *@param out is the output to which any HTML should be sent.
+  *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+  */
+  @Override
+  public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters)
+    throws ManifoldCFException, IOException
+  {
+    String nameNode = parameters.getParameter("namenode");
+    if (nameNode == null) {
+      nameNode = "hdfs://localhost:9000";
+    }
+    
+    String user = parameters.getParameter("user");
+    if (user == null) {
+      user = "user";
+    }
+    
+    out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NameNode") + "</nobr></td>\n"+
+"    <td class=\"value\">\n"+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nameNode)+"</td>\n"+
+"  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.User") + "</nobr></td>\n"+
+"    <td class=\"value\">\n"+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(user)+"</td>\n"+
+"  </tr>\n"+
+"</table>\n"
+    );
+  }
+  
+  /** Output the specification header section.
+  * This method is called in the head section of a job page which has selected a repository connection of the current type.  Its purpose is to add the required tabs
+  * to the list, and to output any javascript methods that might be needed by the job editing HTML.
+  *@param out is the output to which any HTML should be sent.
+  *@param ds is the current document specification for this job.
+  *@param tabsArray is an array of tab names.  Add to this array any tab names that are specific to the connector.
+  */
+  @Override
+  public void outputSpecificationHeader(IHTTPOutput out, Locale locale, DocumentSpecification ds, List<String> tabsArray)
+    throws ManifoldCFException, IOException
+  {
+    tabsArray.add(Messages.getString(locale,"HDFSRepositoryConnector.Paths"));
+    tabsArray.add(Messages.getString(locale,"HDFSRepositoryConnector.FilePathToURITab"));
+
+    out.print(
+"<script type=\"text/javascript\">\n"+
+"<!--\n"+
+"function checkSpecification()\n"+
+"{\n"+
+"  // Does nothing right now.\n"+
+"  return true;\n"+
+"}\n"+
+"\n"+
+"function SpecOp(n, opValue, anchorvalue)\n"+
+"{\n"+
+"  eval(\"editjob.\"+n+\".value = \\\"\"+opValue+\"\\\"\");\n"+
+"  postFormSetAnchor(anchorvalue);\n"+
+"}\n"+
+"//-->\n"+
+"</script>\n"
+    );
+  }
+  
+  /** Output the specification body section.
+  * This method is called in the body section of a job page which has selected a repository connection of the current type.  Its purpose is to present the required form elements for editing.
+  * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags.  The name of the
+  * form is "editjob".
+  *@param out is the output to which any HTML should be sent.
+  *@param ds is the current document specification for this job.
+  *@param tabName is the current tab name.
+  */
+  @Override
+  public void outputSpecificationBody(IHTTPOutput out, Locale locale, DocumentSpecification ds, String tabName)
+    throws ManifoldCFException, IOException
+  {
+    int i;
+    int k;
+
+    // Paths tab
+    if (tabName.equals(Messages.getString(locale,"HDFSRepositoryConnector.Paths")))
+    {
+      out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr><td class=\"separator\" colspan=\"3\"><hr/></td></tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Paths2") + "</nobr></td>\n"+
+"    <td class=\"boxcell\">\n"+
+"      <table class=\"formtable\">\n"+
+"        <tr class=\"formheaderrow\">\n"+
+"          <td class=\"formcolumnheader\"></td>\n"+
+"          <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.RootPath") + "</nobr></td>\n"+
+"          <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Rules") + "</nobr></td>\n"+
+"        </tr>\n"
+      );
+      i = 0;
+      k = 0;
+      while (i < ds.getChildCount())
+      {
+        SpecificationNode sn = ds.getChild(i++);
+        if (sn.getType().equals("startpoint"))
+        {
+          String pathDescription = "_"+Integer.toString(k);
+          String pathOpName = "specop"+pathDescription;
+          out.print(
+"        <tr class=\""+(((k % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+
+"          <td class=\"formcolumncell\">\n"+
+"            <input type=\"hidden\" name=\""+pathOpName+"\" value=\"\"/>\n"+
+"            <input type=\"hidden\" name=\""+"specpath"+pathDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(sn.getAttributeValue("path"))+"\"/>\n"+
+"            <a name=\""+"path_"+Integer.toString(k)+"\">\n"+
+"              <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Delete") + "\" onClick='Javascript:SpecOp(\""+pathOpName+"\",\"Delete\",\"path_"+Integer.toString(k)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.DeletePath")+Integer.toString(k)+"\"/>\n"+
+"            </a>\n"+
+"          </td>\n"+
+"          <td class=\"formcolumncell\">\n"+
+"            <nobr>\n"+
+"              "+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(sn.getAttributeValue("path"))+" \n"+
+"            </nobr>\n"+
+"          </td>\n"+
+"          <td class=\"boxcell\">\n"+
+"            <input type=\"hidden\" name=\""+"specchildcount"+pathDescription+"\" value=\""+Integer.toString(sn.getChildCount())+"\"/>\n"+
+"            <table class=\"formtable\">\n"+
+"              <tr class=\"formheaderrow\">\n"+
+"                <td class=\"formcolumnheader\"></td>\n"+
+"                <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.IncludeExclude") + "</nobr></td>\n"+
+"                <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FileDirectory") + "</nobr></td>\n"+
+"                <td class=\"formcolumnheader\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Match") + "</nobr></td>\n"+
+"              </tr>\n"
+          );
+          int j = 0;
+          while (j < sn.getChildCount())
+          {
+            SpecificationNode excludeNode = sn.getChild(j);
+            String instanceDescription = "_"+Integer.toString(k)+"_"+Integer.toString(j);
+            String instanceOpName = "specop" + instanceDescription;
+
+            String nodeFlavor = excludeNode.getType();
+            String nodeType = excludeNode.getAttributeValue("type");
+            String nodeMatch = excludeNode.getAttributeValue("match");
+            out.print(
+"              <tr class=\"evenformrow\">\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.InsertHere") + "\" onClick='Javascript:SpecOp(\"specop"+instanceDescription+"\",\"Insert Here\",\"match_"+Integer.toString(k)+"_"+Integer.toString(j+1)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.InsertNewMatchForPath")+Integer.toString(k)+" before position #"+Integer.toString(j)+"\"/>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <select name=\""+"specflavor"+instanceDescription+"\">\n"+
+"                      <option value=\"include\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.include") + "</option>\n"+
+"                      <option value=\"exclude\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.exclude") + "</option>\n"+
+"                    </select>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <select name=\""+"spectype"+instanceDescription+"\">\n"+
+"                      <option value=\"file\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.File") + "</option>\n"+
+"                      <option value=\"directory\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Directory") + "</option>\n"+
+"                    </select>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <input type=\"text\" size=\"10\" name=\""+"specmatch"+instanceDescription+"\" value=\"\"/>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"              </tr>\n"+
+"              <tr class=\"oddformrow\">\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <input type=\"hidden\" name=\""+"specop"+instanceDescription+"\" value=\"\"/>\n"+
+"                    <input type=\"hidden\" name=\""+"specfl"+instanceDescription+"\" value=\""+nodeFlavor+"\"/>\n"+
+"                    <input type=\"hidden\" name=\""+"specty"+instanceDescription+"\" value=\""+nodeType+"\"/>\n"+
+"                    <input type=\"hidden\" name=\""+"specma"+instanceDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nodeMatch)+"\"/>\n"+
+"                    <a name=\""+"match_"+Integer.toString(k)+"_"+Integer.toString(j)+"\">\n"+
+"                      <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Delete") + "\" onClick='Javascript:SpecOp(\"specop"+instanceDescription+"\",\"Delete\",\"match_"+Integer.toString(k)+"_"+Integer.toString(j)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.DeletePath")+Integer.toString(k)+", match spec #"+Integer.toString(j)+"\"/>\n"+
+"                    </a>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    "+nodeFlavor+"\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    "+nodeType+"\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    "+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(nodeMatch)+"\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"              </tr>\n"
+            );
+            j++;
+          }
+          if (j == 0)
+          {
+            out.print(
+"              <tr class=\"formrow\"><td class=\"message\" colspan=\"4\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NoRulesDefined") + "</td></tr>\n"
+            );
+          }
+          out.print(
+"              <tr class=\"formrow\"><td class=\"lightseparator\" colspan=\"4\"><hr/></td></tr>\n"+
+"              <tr class=\"formrow\">\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <a name=\""+"match_"+Integer.toString(k)+"_"+Integer.toString(j)+"\">\n"+
+"                    <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Add") + "\" onClick='Javascript:SpecOp(\""+pathOpName+"\",\"Add\",\"match_"+Integer.toString(k)+"_"+Integer.toString(j+1)+"\")' alt=\""+Messages.getAttributeString(locale,"HDFSRepositoryConnector.AddNewMatchForPath")+Integer.toString(k)+"\"/>\n"+
+"                  </a>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <select name=\""+"specflavor"+pathDescription+"\">\n"+
+"                      <option value=\"include\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.include") + "</option>\n"+
+"                      <option value=\"exclude\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.exclude") + "</option>\n"+
+"                    </select>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <select name=\""+"spectype"+pathDescription+"\">\n"+
+"                      <option value=\"file\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.File") + "</option>\n"+
+"                      <option value=\"directory\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.Directory") + "</option>\n"+
+"                    </select>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"                <td class=\"formcolumncell\">\n"+
+"                  <nobr>\n"+
+"                    <input type=\"text\" size=\"10\" name=\""+"specmatch"+pathDescription+"\" value=\"\"/>\n"+
+"                  </nobr>\n"+
+"                </td>\n"+
+"              </tr>\n"+
+"            </table>\n"+
+"          </td>\n"+
+"        </tr>\n"
+          );
+          k++;
+        }
+      }
+      if (k == 0)
+      {
+        out.print(
+"        <tr class=\"formrow\"><td class=\"message\" colspan=\"3\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NoDocumentsSpecified") + "</td></tr>\n"
+        );
+      }
+      out.print(
+"        <tr class=\"formrow\"><td class=\"lightseparator\" colspan=\"3\"><hr/></td></tr>\n"+
+"        <tr class=\"formrow\">\n"+
+"          <td class=\"formcolumncell\">\n"+
+"            <nobr>\n"+
+"              <a name=\""+"path_"+Integer.toString(k)+"\">\n"+
+"                <input type=\"button\" value=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Add") + "\" onClick='Javascript:SpecOp(\"specop\",\"Add\",\"path_"+Integer.toString(i+1)+"\")' alt=\"" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.AddNewPath") + "\"/>\n"+
+"                <input type=\"hidden\" name=\"pathcount\" value=\""+Integer.toString(k)+"\"/>\n"+
+"                <input type=\"hidden\" name=\"specop\" value=\"\"/>\n"+
+"              </a>\n"+
+"            </nobr>\n"+
+"          </td>\n"+
+"          <td class=\"formcolumncell\">\n"+
+"            <nobr>\n"+
+"              <input type=\"text\" size=\"80\" name=\"specpath\" value=\"\"/>\n"+
+"            </nobr>\n"+
+"          </td>\n"+
+"          <td class=\"formcolumncell\">\n"+
+"          </td>\n"+
+"        </tr>\n"+
+"      </table>\n"+
+"    </td>\n"+
+"  </tr>\n"+
+"</table>\n"
+      );
+    }
+    else
+    {
+      i = 0;
+      k = 0;
+      while (i < ds.getChildCount())
+      {
+        SpecificationNode sn = ds.getChild(i++);
+        if (sn.getType().equals("startpoint"))
+        {
+          String pathDescription = "_"+Integer.toString(k);
+          out.print(
+"<input type=\"hidden\" name=\"specpath"+pathDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(sn.getAttributeValue("path"))+"\"/>\n"+
+"<input type=\"hidden\" name=\"specchildcount"+pathDescription+"\" value=\""+Integer.toString(sn.getChildCount())+"\"/>\n"
+          );
+
+          int j = 0;
+	  while (j < sn.getChildCount())
+	  {
+            SpecificationNode excludeNode = sn.getChild(j);
+            String instanceDescription = "_"+Integer.toString(k)+"_"+Integer.toString(j);
+
+            String nodeFlavor = excludeNode.getType();
+            String nodeType = excludeNode.getAttributeValue("type");
+            String nodeMatch = excludeNode.getAttributeValue("match");
+            out.print(
+"<input type=\"hidden\" name=\"specfl"+instanceDescription+"\" value=\""+nodeFlavor+"\"/>\n"+
+"<input type=\"hidden\" name=\"specty"+instanceDescription+"\" value=\""+nodeType+"\"/>\n"+
+"<input type=\"hidden\" name=\"specma"+instanceDescription+"\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(nodeMatch)+"\"/>\n"
+            );
+            j++;
+          }
+          k++;
+        }
+      }
+      out.print(
+"<input type=\"hidden\" name=\"pathcount\" value=\""+Integer.toString(k)+"\"/>\n"
+      );
+    }
+    
+    
+    /*
+     * get filepathtouri value
+     */
+    boolean filePathToUri = false;
+    i = 0;
+    while (i < ds.getChildCount()) {
+      SpecificationNode sn = ds.getChild(i++);
+      if (sn.getType().equals("filepathtouri")) {
+        filePathToUri = Boolean.valueOf(sn.getValue());
+      }
+    }	    
+
+    /*
+     * File path to URI tab
+     */
+    if (tabName.equals(Messages.getString(locale,"HDFSRepositoryConnector.FilePathToURITab"))) {
+      out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr><td colspan=\"2\" class=\"separator\"><hr/></td></tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FilePathToURI") + "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"filepathtouri\" type=\"checkbox\" value=\"true\"" + (filePathToUri ? "checked" : "") +"/>&nbsp;" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FilePathToURIExample") + "\n" +
+"    </td>\n"+
+"  </tr>\n"+
+"</table>\n"
+      );
+    } else {
+      /*
+       * File path to URI tab hiddens
+       */
+      out.print(
+"<input type=\"hidden\" name=\"filepathtouri\" value=\"" + org.apache.manifoldcf.ui.util.Encoder.attributeEscape(Boolean.toString(filePathToUri)) + "\"/>\n"
+      );
+    }
+
+  }
+  
+  /** Process a specification post.
+  * This method is called at the start of job's edit or view page, whenever there is a possibility that form data for a connection has been
+  * posted.  Its purpose is to gather form information and modify the document specification accordingly.
+  * The name of the posted form is "editjob".
+  *@param variableContext contains the post data, including binary file-upload information.
+  *@param ds is the current document specification for this job.
+  *@return null if all is well, or a string error message if there is an error that should prevent saving of the job (and cause a redirection to an error page).
+  */
+  @Override
+  public String processSpecificationPost(IPostParameters variableContext, Locale locale, DocumentSpecification ds)
+    throws ManifoldCFException
+  {
+    String x = variableContext.getParameter("pathcount");
+    if (x != null)
+    {
+      ds.clearChildren();
+      // Find out how many children were sent
+      int pathCount = Integer.parseInt(x);
+      // Gather up these
+      int i = 0;
+      int k = 0;
+      while (i < pathCount)
+      {
+        String pathDescription = "_"+Integer.toString(i);
+        String pathOpName = "specop"+pathDescription;
+        x = variableContext.getParameter(pathOpName);
+        if (x != null && x.equals("Delete"))
+        {
+          // Skip to the next
+          i++;
+          continue;
+        }
+        // Path inserts won't happen until the very end
+        String path = variableContext.getParameter("specpath"+pathDescription);
+        SpecificationNode node = new SpecificationNode("startpoint");
+        node.setAttribute("path",path);
+        // Now, get the number of children
+        String y = variableContext.getParameter("specchildcount"+pathDescription);
+        int childCount = Integer.parseInt(y);
+        int j = 0;
+        int w = 0;
+        while (j < childCount)
+        {
+          String instanceDescription = "_"+Integer.toString(i)+"_"+Integer.toString(j);
+          // Look for an insert or a delete at this point
+          String instanceOp = "specop"+instanceDescription;
+          String z = variableContext.getParameter(instanceOp);
+          String flavor;
+          String type;
+          String match;
+          SpecificationNode sn;
+          if (z != null && z.equals("Delete"))
+          {
+            // Process the deletion as we gather
+            j++;
+            continue;
+          }
+          if (z != null && z.equals("Insert Here"))
+          {
+            // Process the insertion as we gather.
+            flavor = variableContext.getParameter("specflavor"+instanceDescription);
+            type = variableContext.getParameter("spectype"+instanceDescription);
+            match = variableContext.getParameter("specmatch"+instanceDescription);
+            sn = new SpecificationNode(flavor);
+            sn.setAttribute("type",type);
+            sn.setAttribute("match",match);
+            node.addChild(w++,sn);
+          }
+          flavor = variableContext.getParameter("specfl"+instanceDescription);
+          type = variableContext.getParameter("specty"+instanceDescription);
+          match = variableContext.getParameter("specma"+instanceDescription);
+          sn = new SpecificationNode(flavor);
+          sn.setAttribute("type",type);
+          sn.setAttribute("match",match);
+          node.addChild(w++,sn);
+          j++;
+        }
+        if (x != null && x.equals("Add"))
+        {
+          // Process adds to the end of the rules in-line
+          String match = variableContext.getParameter("specmatch"+pathDescription);
+          String type = variableContext.getParameter("spectype"+pathDescription);
+          String flavor = variableContext.getParameter("specflavor"+pathDescription);
+          SpecificationNode sn = new SpecificationNode(flavor);
+          sn.setAttribute("type",type);
+          sn.setAttribute("match",match);
+          node.addChild(w,sn);
+        }
+        ds.addChild(k++,node);
+        i++;
+      }
+
+      // See if there's a global add operation
+      String op = variableContext.getParameter("specop");
+      if (op != null && op.equals("Add"))
+      {
+        String path = variableContext.getParameter("specpath");
+        SpecificationNode node = new SpecificationNode("startpoint");
+        node.setAttribute("path",path);
+        
+        // Now add in the defaults; these will be "include all directories" and "include all files".
+        SpecificationNode sn = new SpecificationNode("include");
+        sn.setAttribute("type","file");
+        sn.setAttribute("match","*");
+        node.addChild(node.getChildCount(),sn);
+        sn = new SpecificationNode("include");
+        sn.setAttribute("type","directory");
+        sn.setAttribute("match","*");
+        node.addChild(node.getChildCount(),sn);
+
+        ds.addChild(k,node);
+      }
+    }
+    
+    /*
+     * "filepathtouri"
+     */
+    String filepathtouri = variableContext.getParameter("filepathtouri");
+    if (filepathtouri != null) {
+      SpecificationNode sn;
+      int i = 0;
+      while (i < ds.getChildCount()) {
+        if (ds.getChild(i).getType().equals("filepathtouri")) {
+          ds.removeChild(i);
+        } else {
+          i++;
+        }
+      }
+      sn = new SpecificationNode("filepathtouri");
+      sn.setValue(filepathtouri);
+      ds.addChild(ds.getChildCount(),sn);
+    }
+    
+    return null;
+  }
+  
+  /** View specification.
+  * This method is called in the body section of a job's view page.  Its purpose is to present the document specification information to the user.
+  * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+  *@param out is the output to which any HTML should be sent.
+  *@param ds is the current document specification for this job.
+  */
+  @Override
+  public void viewSpecification(IHTTPOutput out, Locale locale, DocumentSpecification ds)
+    throws ManifoldCFException, IOException
+  {
+    int i = 0;
+    
+    out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr>\n"+
+"  <td colspan=\"2\" class=\"message\">" + Messages.getAttributeString(locale,"HDFSRepositoryConnector.Paths") + "</td>\n"+
+"  </tr>\n"
+    );
+
+    i = 0;
+    boolean seenAny = false;
+    while (i < ds.getChildCount())
+    {
+      SpecificationNode sn = ds.getChild(i++);
+      if (sn.getType().equals("startpoint"))
+      {
+        if (seenAny == false)
+        {
+          seenAny = true;
+        }
+        out.print(
+"  <tr>\n"+
+"    <td class=\"description\">"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(sn.getAttributeValue("path"))+":"+"</td>\n"+
+"    <td class=\"value\">\n"
+        );
+        int j = 0;
+        while (j < sn.getChildCount())
+        {
+          SpecificationNode excludeNode = sn.getChild(j++);
+          out.print(
+"      "+(excludeNode.getType().equals("include")?"Include ":"")+"\n"+
+"      "+(excludeNode.getType().equals("exclude")?"Exclude ":"")+"\n"+
+"      "+(excludeNode.getAttributeValue("type").equals("file")?"file ":"")+"\n"+
+"      "+(excludeNode.getAttributeValue("type").equals("directory")?"directory ":"")+"\n"+
+"      "+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(excludeNode.getAttributeValue("match"))+"<br/>\n"
+          );
+        }
+        out.print(
+"    </td>\n"+
+"  </tr>\n"
+        );
+      }
+    }
+    if (seenAny == false)
+    {
+      out.print(
+"  <tr><td class=\"message\">" + Messages.getBodyString(locale,"HDFSRepositoryConnector.NoDocumentsSpecified") + "</td></tr>\n"
+      );
+    }
+    out.print(
+"</table>\n"
+    );
+    
+    /*
+     * get filepathtouri value
+     */
+    boolean filePathToUri = false;
+    i = 0;
+    while (i < ds.getChildCount()) {
+      SpecificationNode sn = ds.getChild(i++);
+      if (sn.getType().equals("filepathtouri")) {
+        filePathToUri = Boolean.valueOf(sn.getValue());
+      }
+    }
+
+    out.print(
+"<table class=\"displaytable\">\n"+
+"  <tr><td colspan=\"2\" class=\"separator\"><hr/></td></tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"HDFSRepositoryConnector.FilePathToURI") + "</nobr></td>\n"+
+"    <td class=\"value\">" + org.apache.manifoldcf.ui.util.Encoder.bodyEscape(Boolean.toString(filePathToUri)) + "</td>\n"+
+"  </tr>\n"+
+"</table>\n"
+    );
+  
+  }
+
+  // Protected static methods
+
+  /** Check if a file or directory should be included, given a document specification.
+  *@param fileName is the canonical file name.
+  *@param documentSpecification is the specification.
+  *@return true if it should be included.
+  */
+  protected static boolean checkInclude(String nameNode, FileStatus fileStatus, String fileName, DocumentSpecification documentSpecification)
+    throws ManifoldCFException
+  {
+    /*
+     * TODO:
+     * fileName = hdfs://localhost:9000/user/minoru/KEN_ALL_UTF-8_UNIX_SHRINK.CSV
+     * pathPart = hdfs://localhost:9000/user/minoru
+     * fliePart = KEN_ALL_UTF-8_UNIX_SHRINK.CSV
+     * path = /user/minoru => hdfs://localhost:9000/user/minoru
+     */
+    if (Logging.connectors.isDebugEnabled())
+    {
+      Logging.connectors.debug("Checking whether to include file '"+fileName+"'");
+    }
+
+    String pathPart;
+    String filePart;
+    if (fileStatus.isDir())
+    {
+      pathPart = fileName;
+      filePart = null;
+    }
+    else
+    {
+      pathPart = fileStatus.getPath().getParent().toString();
+      filePart = fileStatus.getPath().getName();
+    }
+
+    // Scan until we match a startpoint
+    int i = 0;
+    while (i < documentSpecification.getChildCount())
+    {
+      SpecificationNode sn = documentSpecification.getChild(i++);
+      if (sn.getType().equals("startpoint"))
+      {
+        String path = null;
+        try {
+			path = new URI(nameNode).resolve(sn.getAttributeValue("path")).toString();
+		} catch (URISyntaxException e) {
+			e.printStackTrace();
+		}
+        if (Logging.connectors.isDebugEnabled())
+        {
+          Logging.connectors.debug("Checking path '"+path+"' against canonical '"+pathPart+"'");
+        }
+        // Compare with filename
+        int matchEnd = matchSubPath(path,pathPart);
+        if (matchEnd == -1)
+        {
+          if (Logging.connectors.isDebugEnabled())
+          {
+            Logging.connectors.debug("Match check '"+path+"' against canonical '"+pathPart+"' failed");
+          }
+
+          continue;
+        }
+        // matchEnd is the start of the rest of the path (after the match) in fileName.
+        // We need to walk through the rules and see whether it's in or out.
+        int j = 0;
+        while (j < sn.getChildCount())
+        {
+          SpecificationNode node = sn.getChild(j++);
+          String flavor = node.getType();
+          String match = node.getAttributeValue("match");
+          String type = node.getAttributeValue("type");
+          // If type is "file", then our match string is against the filePart.
+          // If filePart is null, then this rule is simply skipped.
+          String sourceMatch;
+          int sourceIndex;
+          if (type.equals("file"))
+          {
+            if (filePart == null)
+            {
+              continue;
+            }
+            sourceMatch = filePart;
+            sourceIndex = 0;
+          }
+          else
+          {
+            if (filePart != null)
+            {
+              continue;
+            }
+            sourceMatch = pathPart;
+            sourceIndex = matchEnd;
+          }
+
+          if (flavor.equals("include"))
+          {
+            if (checkMatch(sourceMatch,sourceIndex,match))
+            {
+              return true;
+            }
+          }
+          else if (flavor.equals("exclude"))
+          {
+            if (checkMatch(sourceMatch,sourceIndex,match))
+            {
+              return false;
+            }
+          }
+        }
+      }
+    }
+    if (Logging.connectors.isDebugEnabled())
+    {
+      Logging.connectors.debug("Not including '"+fileName+"' because no matching rules");
+    }
+
+    return false;
+  }
+
+  /** Check if a file should be ingested, given a document specification.  It is presumed that
+  * documents that do not pass checkInclude() will be checked with this method.
+  *@param file is the file.
+  *@param documentSpecification is the specification.
+  */
+  protected static boolean checkIngest(String nameNode, FileStatus fileStatus, DocumentSpecification documentSpecification)
+    throws ManifoldCFException
+  {
+    // Since the only exclusions at this point are not based on file contents, this is a no-op.
+    // MHL
+    return true;
+  }
+
+  /** Match a sub-path.  The sub-path must match the complete starting part of the full path, in a path
+  * sense.  The returned value should point into the file name beyond the end of the matched path, or
+  * be -1 if there is no match.
+  *@param subPath is the sub path.
+  *@param fullPath is the full path.
+  *@return the index of the start of the remaining part of the full path, or -1.
+  */
+  protected static int matchSubPath(String subPath, String fullPath)
+  {
+    if (subPath.length() > fullPath.length())
+      return -1;
+    if (fullPath.startsWith(subPath) == false)
+      return -1;
+    int rval = subPath.length();
+    if (fullPath.length() == rval)
+      return rval;
+    char x = fullPath.charAt(rval);
+    if (x == Path.SEPARATOR_CHAR)
+      rval++;
+    return rval;
+  }
+
+  /** Check a match between two strings with wildcards.
+  *@param sourceMatch is the expanded string (no wildcards)
+  *@param sourceIndex is the starting point in the expanded string.
+  *@param match is the wildcard-based string.
+  *@return true if there is a match.
+  */
+  protected static boolean checkMatch(String sourceMatch, int sourceIndex, String match)
+  {
+    // Note: The java regex stuff looks pretty heavyweight for this purpose.
+    // I've opted to try and do a simple recursive version myself, which is not compiled.
+    // Basically, the match proceeds by recursive descent through the string, so that all *'s cause
+    // recursion.
+    boolean caseSensitive = true;
+
+    return processCheck(caseSensitive, sourceMatch, sourceIndex, match, 0);
+  }
+
+  /** Recursive worker method for checkMatch.  Returns 'true' if there is a path that consumes both
+  * strings in their entirety in a matched way.
+  *@param caseSensitive is true if file names are case sensitive.
+  *@param sourceMatch is the source string (w/o wildcards)
+  *@param sourceIndex is the current point in the source string.
+  *@param match is the match string (w/wildcards)
+  *@param matchIndex is the current point in the match string.
+  *@return true if there is a match.
+  */
+  protected static boolean processCheck(boolean caseSensitive, String sourceMatch, int sourceIndex,
+    String match, int matchIndex)
+  {
+    // Logging.connectors.debug("Matching '"+sourceMatch+"' position "+Integer.toString(sourceIndex)+
+    //      " against '"+match+"' position "+Integer.toString(matchIndex));
+
+    // Match up through the next * we encounter
+    while (true)
+    {
+      // If we've reached the end, it's a match.
+      if (sourceMatch.length() == sourceIndex && match.length() == matchIndex)
+        return true;
+      // If one has reached the end but the other hasn't, no match
+      if (match.length() == matchIndex)
+        return false;
+      if (sourceMatch.length() == sourceIndex)
+      {
+        if (match.charAt(matchIndex) != '*')
+          return false;
+        matchIndex++;
+        continue;
+      }
+      char x = sourceMatch.charAt(sourceIndex);
+      char y = match.charAt(matchIndex);
+      if (!caseSensitive)
+      {
+        if (x >= 'A' && x <= 'Z')
+          x -= 'A'-'a';
+        if (y >= 'A' && y <= 'Z')
+          y -= 'A'-'a';
+      }
+      if (y == '*')
+      {
+        // Wildcard!
+        // We will recurse at this point.
+        // Basically, we want to combine the results for leaving the "*" in the match string
+        // at this point and advancing the source index, with skipping the "*" and leaving the source
+        // string alone.
+        return processCheck(caseSensitive,sourceMatch,sourceIndex+1,match,matchIndex) ||
+          processCheck(caseSensitive,sourceMatch,sourceIndex,match,matchIndex+1);
+      }
+      if (y == '?' || x == y)
+      {
+        sourceIndex++;
+        matchIndex++;
+      }
+      else
+        return false;
+    }
+  }
+
+  /** Document identifier stream.
+  */
+  protected static class IdentifierStream implements IDocumentIdentifierStream
+  {
+    protected String[] ids = null;
+    protected int currentIndex = 0;
+
+    public IdentifierStream(DocumentSpecification spec)
+      throws ManifoldCFException
+    {
+      // Walk the specification for the "startpoint" types.  Amalgamate these into a list of strings.
+      // Presume that all roots are startpoint nodes
+      int i = 0;
+      int j = 0;
+      while (i < spec.getChildCount())
+      {
+        SpecificationNode n = spec.getChild(i);
+        if (n.getType().equals("startpoint"))
+        {
+          j++;
+        }
+        i++;
+      }
+      ids = new String[j];
+      i = 0;
+      j = 0;
+      while (i < ids.length)
+      {
+        SpecificationNode n = spec.getChild(i);
+        if (n.getType().equals("startpoint"))
+        {
+          // The id returned MUST be in canonical form!!!
+          ids[j] = new Path(n.getAttributeValue("path")).toString();
+          if (Logging.connectors.isDebugEnabled())
+          {
+            Logging.connectors.debug("Seed = '"+ids[j]+"'");
+          }
+          j++;
+        }
+        i++;
+      }
+    }
+
+    /** Get the next identifier.
+    *@return the next document identifier, or null if there are no more.
+    */
+    public String getNextIdentifier()
+      throws ManifoldCFException, ServiceInterruption
+    {
+      if (currentIndex == ids.length)
+        return null;
+      return ids[currentIndex++];
+    }
+
+    /** Close the stream.
+    */
+    public void close()
+      throws ManifoldCFException
+    {
+      ids = null;
+    }
+
+  }
+
+}

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,141 @@
+/* $Id: Messages.java 1295926 2012-03-01 21:56:27Z kwright $ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.hdfs;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+  public static final String DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.crawler.connectors.hdfs.common";
+  public static final String DEFAULT_PATH_NAME="org.apache.manifoldcf.crawler.connectors.hdfs";
+  
+  /** Constructor - do no instantiate
+  */
+  protected Messages()
+  {
+  }
+  
+  public static String getString(Locale locale, String messageKey)
+  {
+    return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getAttributeString(Locale locale, String messageKey)
+  {
+    return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getBodyString(Locale locale, String messageKey)
+  {
+    return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getAttributeJavascriptString(Locale locale, String messageKey)
+  {
+    return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getBodyJavascriptString(Locale locale, String messageKey)
+  {
+    return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+  }
+
+  public static String getString(Locale locale, String messageKey, Object[] args)
+  {
+    return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getAttributeString(Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+  
+  public static String getBodyString(Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getAttributeJavascriptString(Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  public static String getBodyJavascriptString(Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+  }
+
+  // More general methods which allow bundlenames and class loaders to be specified.
+  
+  public static String getString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getAttributeString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getBodyString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+  }
+  
+  public static String getAttributeJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getAttributeJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  public static String getBodyJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+  {
+    return getBodyJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+  }
+
+  // Resource output
+  
+  public static void outputResource(IHTTPOutput output, Locale locale, String resourceKey,
+    Map<String,String> substitutionParameters, boolean mapToUpperCase)
+    throws ManifoldCFException
+  {
+    outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+      substitutionParameters,mapToUpperCase);
+  }
+  
+  public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+    Map<String,String> substitutionParameters, boolean mapToUpperCase)
+    throws ManifoldCFException
+  {
+    outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+      substitutionParameters,mapToUpperCase);
+  }
+
+  public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+    Map<String,Object> contextObjects)
+    throws ManifoldCFException
+  {
+    outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+      contextObjects);
+  }
+  
+}
+

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties Mon Jun 24 03:26:56 2013
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDFSOutputConnector.ServerTabName=Server
+HDFSOutputConnector.NameNode=Name Node:
+HDFSOutputConnector.User=User:
+
+HDFSOutputConnector.PathTabName=Output Path
+HDFSOutputConnector.Path=Output Path:
+HDFSOutputConnector.RootPath=Root path:
+

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties Mon Jun 24 03:26:56 2013
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDFSOutputConnector.ServerTabName=サーバー
+HDFSOutputConnector.NameNode=ネームノード:
+HDFSOutputConnector.User=ユーザー:
+
+HDFSOutputConnector.PathTabName=出力パス
+HDFSOutputConnector.Path=出力パス:
+HDFSOutputConnector.RootPath=ルートパス:

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties Mon Jun 24 03:26:56 2013
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDFSRepositoryConnector.ServerTabName=Server
+HDFSRepositoryConnector.NameNode=Name Node:
+HDFSRepositoryConnector.User=User:
+
+HDFSRepositoryConnector.Paths=Repository Paths
+HDFSRepositoryConnector.Paths2=Repository Paths:
+HDFSRepositoryConnector.RootPath=Root path
+HDFSRepositoryConnector.Rules=Rules
+HDFSRepositoryConnector.Delete=Delete
+HDFSRepositoryConnector.DeletePath=Delete path #
+HDFSRepositoryConnector.IncludeExclude=Include/exclude
+HDFSRepositoryConnector.FileDirectory=File/directory
+HDFSRepositoryConnector.Match=Match
+HDFSRepositoryConnector.include=include
+HDFSRepositoryConnector.exclude=exclude
+HDFSRepositoryConnector.File=File
+HDFSRepositoryConnector.Directory=Directory
+HDFSRepositoryConnector.NoDocumentsSpecified=No documents specified
+HDFSRepositoryConnector.Add=Add
+HDFSRepositoryConnector.InsertHere=Insert Here
+HDFSRepositoryConnector.include=include
+HDFSRepositoryConnector.NoRulesDefined=No rules defined
+HDFSRepositoryConnector.InsertNewMatchForPath=Insert new match for path #
+HDFSRepositoryConnector.DeletePath=Delete path #
+HDFSRepositoryConnector.AddNewMatchForPath=Add new match for path #
+HDFSRepositoryConnector.AddNewPath=Add new path
+
+HDFSRepositoryConnector.FilePathToURITab=Convert file path to URI
+HDFSRepositoryConnector.FilePathToURI=Convert file path to URI:
+HDFSRepositoryConnector.FilePathToURIExample=EX) $REPOSITORY_PATH/http/localhost/index.html => http://localhost/index.html

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message