manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1649598 - in /manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src: main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ test/ test/java/ test/java/org/ test/java/org/apache/ test/java/org/apache/man...
Date Mon, 05 Jan 2015 17:49:12 GMT
Author: kwright
Date: Mon Jan  5 17:49:11 2015
New Revision: 1649598

URL: http://svn.apache.org/r1649598
Log:
Add tests of expression evaluation, and make some methods public for that reason

Added:
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
  (with props)
Modified:
    manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java

Modified: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java?rev=1649598&r1=1649597&r2=1649598&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
(original)
+++ manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
Mon Jan  5 17:49:11 2015
@@ -114,7 +114,12 @@ public class ForcedMetadataConnector ext
         }
       } else if (sp.keepAllMetadata()) {
         // Copy ALL current fields from old document, but go through FieldDataFactory
-        // MHL
+        Iterator<String> fields = document.getFields();
+        while (fields.hasNext())
+        {
+          String field = fields.next();
+          moveData(docCopy,field,fdf,field,false);
+        }
       }
       
       // Iterate through the expressions
@@ -624,7 +629,7 @@ public class ForcedMetadataConnector ext
     return input;
   }
   
-  protected interface IDataSource {
+  public interface IDataSource {
     public int getSize() throws IOException, ManifoldCFException;
     public Object[] getRawForm() throws IOException, ManifoldCFException;
     public String[] getStringForm() throws IOException, ManifoldCFException;
@@ -712,7 +717,7 @@ public class ForcedMetadataConnector ext
     return new StringSource(rval);
   }
   
-  protected static IDataSource processExpression(String expression, FieldDataFactory sourceDocument)
+  public static IDataSource processExpression(String expression, FieldDataFactory sourceDocument)
     throws IOException, ManifoldCFException {
     int index = 0;
     IDataSource input = null;
@@ -874,7 +879,7 @@ public class ForcedMetadataConnector ext
   * a CharacterInput implementation, thus making a temporary file copy.  So it is imperative
   * that this object is closed when it is no longer needed.
   */
-  protected static class FieldDataFactory {
+  public static class FieldDataFactory {
     
     protected final RepositoryDocument sourceDocument;
     

Added: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java?rev=1649598&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
(added)
+++ manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
Mon Jan  5 17:49:11 2015
@@ -0,0 +1,89 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.webcrawler;
+
+import org.apache.manifoldcf.agents.transformation.forcedmetadata.*;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import java.io.*;
+import java.util.*;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+public class ExpressionTest
+{
+
+  @Test
+  public void simpleExpressions()
+    throws Exception {
+    RepositoryDocument inputDoc = new RepositoryDocument();
+    inputDoc.addField("stringfield",new String[]{"stringa","stringb","stringc"});
+    inputDoc.addField("readerfield",new Reader[]{new StringReader("readera"),new StringReader("readerb")});
+    inputDoc.addField("datefield",new Date[]{new Date(0L), new Date(100000000L)});
+    ForcedMetadataConnector.FieldDataFactory fdf = new ForcedMetadataConnector.FieldDataFactory(inputDoc);
+    try {
+      arrayEquals(new String[]{"stringa","stringb","stringc"}, (String[])(ForcedMetadataConnector.processExpression("${stringfield}",
fdf).getRawForm()));
+      arrayEquals(new String[]{"prefixstringapostfix","prefixstringbpostfix","prefixstringcpostfix"},
(String[])(ForcedMetadataConnector.processExpression("prefix${stringfield}postfix", fdf).getRawForm()));
+      arrayEquals(new Reader[]{new StringReader("readera"),new StringReader("readerb")},
(Reader[])(ForcedMetadataConnector.processExpression("${readerfield}", fdf).getRawForm()));
+      // Second access of reader fields, without prior string conversion, also must work
+      arrayEquals(new Reader[]{new StringReader("readera"),new StringReader("readerb")},
(Reader[])(ForcedMetadataConnector.processExpression("${readerfield}", fdf).getRawForm()));
+      arrayEquals(new String[]{"prefixreaderapostfix","prefixreaderbpostfix"}, (String[])(ForcedMetadataConnector.processExpression("prefix${readerfield}postfix",
fdf).getRawForm()));
+    } finally {
+      fdf.close();
+    }
+  }
+  
+  protected static void arrayEquals(Reader[] expected, Reader[] actual)
+    throws Exception {
+    assertEquals(expected.length,actual.length);
+    Set<String> expectedSet = new HashSet<String>();
+    for (Reader expectedValue : expected) {
+      expectedSet.add(readData(expectedValue));
+    }
+    for (Reader actualValue : actual) {
+      assertEquals(true,expectedSet.contains(readData(actualValue)));
+    }
+
+  }
+  
+  protected static String readData(Reader r)
+    throws IOException {
+    StringBuilder sb = new StringBuilder();
+    char[] buffer = new char[65536];
+    while (true) {
+      int amt = r.read(buffer);
+      if (amt == -1)
+        break;
+      sb.append(buffer,0,amt);
+    }
+    return sb.toString();
+  }
+  
+  protected static void arrayEquals(String[] expected, String[] actual)
+    throws Exception {
+    assertEquals(expected.length,actual.length);
+    Set<String> expectedSet = new HashSet<String>();
+    for (String expectedValue : expected) {
+      expectedSet.add(expectedValue);
+    }
+    for (String actualValue : actual) {
+      assertEquals(true,expectedSet.contains(actualValue));
+    }
+  }
+}

Propchange: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
------------------------------------------------------------------------------
    svn:keywords = Id



Mime
View raw message