manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1642540 - in /manifoldcf/trunk: ./ connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/ connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
Date Sun, 30 Nov 2014 12:54:06 GMT
Author: kwright
Date: Sun Nov 30 12:54:06 2014
New Revision: 1642540

URL: http://svn.apache.org/r1642540
Log:
Fix for CONNECTORS-969.

Removed:
    manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputSpecs.java
    manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java
Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
    manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1642540&r1=1642539&r2=1642540&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Nov 30 12:54:06 2014
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 2.0-dev =====================
 
+CONNECTORS-969: Don't use JSON as an output specification format.
+(Karl Wright)
+
 CONNECTORS-974: Make SharePoint 2010 be the default selection.
 (Karl Wright)
 

Modified: manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java?rev=1642540&r1=1642539&r2=1642540&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
(original)
+++ manifoldcf/trunk/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
Sun Nov 30 12:54:06 2014
@@ -24,6 +24,10 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.StringReader;
+
+import org.apache.commons.io.IOUtils;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -35,6 +39,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.manifoldcf.agents.interfaces.*;
 import org.apache.manifoldcf.agents.output.BaseOutputConnector;
@@ -142,7 +148,7 @@ public class FileOutputConnector extends
   @Override
   public VersionContext getPipelineDescription(Specification spec) throws ManifoldCFException,
ServiceInterruption {
     FileOutputSpecs specs = new FileOutputSpecs(getSpecNode(spec));
-    return new VersionContext(specs.toJson().toString(),params,spec);
+    return new VersionContext(specs.toVersionString(),params,spec);
   }
 
   /** Add (or replace) a document in the output data store using the connector.
@@ -686,4 +692,98 @@ public class FileOutputConnector extends
     }
     return sb.toString();
   }
+  
+  protected static class FileOutputSpecs extends FileOutputParam {
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 1859652730572662025L;
+
+    final public static ParameterEnum[] SPECIFICATIONLIST = {
+      ParameterEnum.ROOTPATH
+    };
+
+    private final String rootPath;
+
+    /** Build a set of ElasticSearch parameters by reading an instance of
+     * SpecificationNode.
+     * 
+     * @param node
+     * @throws ManifoldCFException
+     */
+    public FileOutputSpecs(ConfigurationNode node) throws ManifoldCFException {
+      super(SPECIFICATIONLIST);
+      String rootPath = null;
+      for (ParameterEnum param : SPECIFICATIONLIST) {
+        String value = null;
+        if (node != null) {
+          value = node.getAttributeValue(param.name());
+        }
+        if (value == null) {
+          value = param.defaultValue;
+        }
+        put(param, value);
+      }
+      rootPath = getRootPath();
+      this.rootPath = rootPath;
+    }
+
+    /**
+     * @param variableContext
+     * @param specNode
+     */
+    public static void contextToSpecNode(IPostParameters variableContext, ConfigurationNode
specNode, int sequenceNumber) {
+      for (ParameterEnum param : SPECIFICATIONLIST) {
+        String p = variableContext.getParameter("s"+sequenceNumber+"_"+param.name().toLowerCase());
+        if (p != null) {
+          specNode.setAttribute(param.name(), p);
+        }
+      }
+    }
+
+    /** @return a version string representation of the parameter list */
+    public String toVersionString() {
+      StringBuilder sb = new StringBuilder();
+      pack(sb,rootPath,'+');
+      return sb.toString();
+    }
+
+    /**
+     * @return
+     */
+    public String getRootPath() {
+      return get(ParameterEnum.ROOTPATH);
+    }
+
+    /**
+     * @param content
+     * @return
+     * @throws ManifoldCFException
+     */
+    private final static TreeSet<String> createStringSet(String content) throws ManifoldCFException
{
+      TreeSet<String> set = new TreeSet<String>();
+      BufferedReader br = null;
+      StringReader sr = null;
+      try {
+        sr = new StringReader(content);
+        br = new BufferedReader(sr);
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          line = line.trim();
+          if (line.length() > 0) {
+            set.add(line);
+          }
+        }
+        return set;
+      } catch (IOException e) {
+        throw new ManifoldCFException(e);
+      } finally {
+        if (br != null) {
+          IOUtils.closeQuietly(br);
+        }
+      }
+    }
+
+  }
+
 }

Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1642540&r1=1642539&r2=1642540&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Sun Nov 30 12:54:06 2014
@@ -25,12 +25,18 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.StringReader;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.io.IOUtils;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -252,7 +258,7 @@ public class HDFSOutputConnector extends
   @Override
   public VersionContext getPipelineDescription(Specification spec) throws ManifoldCFException,
ServiceInterruption {
     HDFSOutputSpecs specs = new HDFSOutputSpecs(getSpecNode(spec));
-    return new VersionContext(specs.toJson().toString(),params,spec);
+    return new VersionContext(specs.toVersionString(),params,spec);
   }
 
   /** Add (or replace) a document in the output data store using the connector.
@@ -323,9 +329,6 @@ public class HDFSOutputConnector extends
       Long startTime = new Long(System.currentTimeMillis());
       deleteFile(path,activities,documentURI);
       activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", null);
-    } catch (JSONException e) {
-      activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to delete document due to: " + e.getMessage());
-      handleJSONException(e);
     } catch (URISyntaxException e) {
       activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,e.getClass().getSimpleName().toUpperCase(Locale.ROOT),"Failed
to delete document due to: " + e.getMessage());
       handleURISyntaxException(e);
@@ -855,4 +858,112 @@ public class HDFSOutputConnector extends
     }
   }
 
+  public static class HDFSOutputSpecs extends HDFSOutputParam {
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 1145652730572662025L;
+
+    final public static ParameterEnum[] SPECIFICATIONLIST = {
+      ParameterEnum.rootpath
+    };
+
+    private final String rootPath;
+
+    /** Build a set of ElasticSearch parameters by reading an JSON object
+     * 
+     * @param json
+     * @throws JSONException
+     * @throws ManifoldCFException
+     */
+    public HDFSOutputSpecs(String versionString) throws ManifoldCFException {
+      super(SPECIFICATIONLIST);
+      int index = 0;
+      StringBuilder rootPathBuffer = new StringBuilder();
+      index = unpack(rootPathBuffer,versionString,index,'+');
+      this.rootPath = rootPathBuffer.toString();
+      // MHL
+    }
+
+    /** Build a set of ElasticSearch parameters by reading an instance of
+     * SpecificationNode.
+     * 
+     * @param node
+     * @throws ManifoldCFException
+     */
+    public HDFSOutputSpecs(ConfigurationNode node) throws ManifoldCFException {
+      super(SPECIFICATIONLIST);
+      String rootPath = null;
+      for (ParameterEnum param : SPECIFICATIONLIST) {
+        String value = null;
+        if (node != null) {
+          value = node.getAttributeValue(param.name());
+        }
+        if (value == null) {
+          value = param.defaultValue;
+        }
+        put(param, value);
+      }
+      rootPath = getRootPath();
+      this.rootPath = rootPath;
+    }
+
+    /**
+      * @param variableContext
+      * @param specNode
+      */
+    public static void contextToSpecNode(IPostParameters variableContext, ConfigurationNode
specNode, int sequenceNumber) {
+      for (ParameterEnum param : SPECIFICATIONLIST) {
+        String p = variableContext.getParameter("s"+sequenceNumber+"_"+param.name().toLowerCase());
+        if (p != null) {
+          specNode.setAttribute(param.name(), p);
+        }
+      }
+    }
+
+    /** @return a JSON representation of the parameter list */
+    public String toVersionString() {
+      StringBuilder sb = new StringBuilder();
+      pack(sb,rootPath,'+');
+      return sb.toString();
+    }
+
+    /**
+     * @return
+     */
+    public String getRootPath() {
+      return get(ParameterEnum.rootpath);
+    }
+
+    /**
+     * @param content
+     * @return
+     * @throws ManifoldCFException
+     */
+    private final static TreeSet<String> createStringSet(String content) throws ManifoldCFException
{
+      TreeSet<String> set = new TreeSet<String>();
+      BufferedReader br = null;
+      StringReader sr = null;
+      try {
+        sr = new StringReader(content);
+        br = new BufferedReader(sr);
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          line = line.trim();
+          if (line.length() > 0) {
+            set.add(line);
+          }
+        }
+        return set;
+      } catch (IOException e) {
+        throw new ManifoldCFException(e.getMessage(),e);
+      } finally {
+        if (br != null) {
+          IOUtils.closeQuietly(br);
+        }
+      }
+    }
+
+  }
+
 }



Mime
View raw message