manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1649608 - in /manifoldcf/branches/dev_1x: ./ connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agen...
Date Mon, 05 Jan 2015 18:40:19 GMT
Author: kwright
Date: Mon Jan  5 18:40:18 2015
New Revision: 1649608

URL: http://svn.apache.org/r1649608
Log:
Pull up fix for CONNECTORS-1134 from trunk.

Added:
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_Expressions.html
      - copied unchanged from r1649605, manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_Expressions.html
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/test/
      - copied from r1649605, manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/
Removed:
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_ForcedMetadata.html
Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/CHANGES.txt
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js
    manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html
    manifoldcf/branches/dev_1x/framework/   (props changed)
    manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java
    manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan  5 18:40:18 2015
@@ -46,6 +46,7 @@
 /manifoldcf/branches/CONNECTORS-1118:1644108-1644398
 /manifoldcf/branches/CONNECTORS-1119:1645497-1647584
 /manifoldcf/branches/CONNECTORS-1130:1648217-1648685
+/manifoldcf/branches/CONNECTORS-1134:1649449-1649604
 /manifoldcf/branches/CONNECTORS-120:1406712-1407974,1407982-1411043,1411049-1416451
 /manifoldcf/branches/CONNECTORS-120-1:1416450-1417056
 /manifoldcf/branches/CONNECTORS-13:1525862-1527182,1539324-1541634
@@ -120,4 +121,4 @@
 /manifoldcf/branches/CONNECTORS-981:1605049-1605773
 /manifoldcf/branches/CONNECTORS-989:1611600-1612101
 /manifoldcf/branches/CONNECTORS-990:1610284-1610707
-/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
 4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529
+/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
 4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605

Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Mon Jan  5 18:40:18 2015
@@ -3,6 +3,12 @@ $Id$
 
 ======================= 1.9-dev =====================
 
+CONNECTORS-1134: Revamp metadata adjuster transformer to
+handle arbitrary combination expressions across multiple fields.
+This replaces the "move" and "add" paradigm with a more general
+approach.
+(Hemant Jain, Karl Wright)
+
 CONNECTORS-1135: Update Japanese translations for JCIFS connector.
 (KOIZUMI Satoru)
 

Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java Mon Jan  5 18:40:18 2015
@@ -34,6 +34,7 @@ public class ForcedMetadataConnector ext
   // Nodes and attributes representing parameters and values.
   // There will be node for every parameter/value pair.
   
+  public static final String NODE_EXPRESSION = "expression";
   public static final String NODE_PAIR = "pair";
   public static final String ATTRIBUTE_PARAMETER = "parameter";
   public static final String NODE_FIELDMAP = "fieldmap";
@@ -47,8 +48,7 @@ public class ForcedMetadataConnector ext
   
   private static final String VIEW_SPEC = "viewSpecification.html";
   private static final String EDIT_SPEC_HEADER = "editSpecification.js";
-  private static final String EDIT_SPEC_FORCED_METADATA = "editSpecification_ForcedMetadata.html";
-  private static final String EDIT_SPEC_FIELDMAPPING = "editSpecification_FieldMapping.html";
+  private static final String EDIT_SPEC_EXPRESSIONS = "editSpecification_Expressions.html";
 
   /** Get a pipeline version string, given a pipeline specification object.  The version string is used to
   * uniquely describe the pertinent details of the specification and the configuration, to allow the Connector 
@@ -91,41 +91,135 @@ public class ForcedMetadataConnector ext
   {
     // Unpack the forced metadata
     SpecPacker sp = new SpecPacker(pipelineDescription.getSpecification());
-    // We have to create a copy of the Repository Document, since we might be rearranging things
-    RepositoryDocument docCopy = document.duplicate();
-    docCopy.clearFields();
-    // Do the mapping first!!
-    Iterator<String> fields = document.getFields();
-    while (fields.hasNext())
-    {
-      String field = fields.next();
-      String target = sp.getMapping(field);
-      if (target != null)
-      {
-        moveData(docCopy,target,document,field,sp.filterEmpty());
-      }
-      else
-      {
-        if (sp.keepAllMetadata())
+    
+    // Create a structure that will allow us access to fields without sharing Reader objects
+    FieldDataFactory fdf = new FieldDataFactory(document);
+    try {
+      // We have to create a copy of the Repository Document, since we might be rearranging things
+      RepositoryDocument docCopy = document.duplicate();
+      // We must explicitly copy all fields, since we can't share references to Reader objects and
+      // expect anything to work right
+      docCopy.clearFields();
+      
+      // Clear fields, unless we're supposed to keep what we don't specify
+      if (sp.filterEmpty()) {
+        if (sp.keepAllMetadata()) {
+          // Loop through fields and copy them, filtering empties
+          Iterator<String> fields = document.getFields();
+          while (fields.hasNext())
+          {
+            String field = fields.next();
+            moveData(docCopy,field,fdf,field,true);
+          }
+        }
+      } else if (sp.keepAllMetadata()) {
+        // Copy ALL current fields from old document, but go through FieldDataFactory
+        Iterator<String> fields = document.getFields();
+        while (fields.hasNext())
         {
-          moveData(docCopy,field,document,field,sp.filterEmpty());
+          String field = fields.next();
+          moveData(docCopy,field,fdf,field,false);
         }
       }
+      
+      // Iterate through the expressions
+      Iterator<String> expressionKeys = sp.getExpressionKeys();
+      while (expressionKeys.hasNext()) {
+        String expressionKey = expressionKeys.next();
+        // Get the set of expressions for the key
+        Set<String> values = sp.getExpressionValues(expressionKey);
+        IDataSource[] dataSources = new IDataSource[values.size()];
+        int k = 0;
+        for (String expression : values) {
+          dataSources[k++] = processExpression(expression, fdf);
+        }
+        int totalSize = 0;
+        for (IDataSource dataSource : dataSources) {
+          if (dataSource != null)
+            totalSize += dataSource.getSize();
+        }
+        if (totalSize == 0) {
+          docCopy.removeField(expressionKey);
+        } else {
+          // Each IDataSource will contribute zero or more results to the final array.  But here's the tricky part:
+          // the results all must be of the same type.  If there are any differences, then we have to bash them all to
+          // strings first.
+          Object[] allValues;
+          k = 0;
+          if (allDates(dataSources)) {
+            allValues = new Date[totalSize];
+            for (IDataSource dataSource : dataSources) {
+              if (dataSource != null) {
+                for (Object o : dataSource.getRawForm()) {
+                  allValues[k++] = o;
+                }
+              }
+            }
+            docCopy.addField(expressionKey,(Date[])conditionallyRemoveNulls(allValues,sp.filterEmpty()));
+          } else if (allReaders(dataSources)) {
+            if (sp.filterEmpty())
+              allValues = new String[totalSize];
+            else
+              allValues = new Reader[totalSize];
+            for (IDataSource dataSource : dataSources) {
+              if (dataSource != null) {
+                Object[] sources = sp.filterEmpty()?dataSource.getStringForm():dataSource.getRawForm();
+                for (Object o : sources) {
+                  allValues[k++] = o;
+                }
+              }
+            }
+            if (sp.filterEmpty())
+              docCopy.addField(expressionKey,removeEmpties((String[])allValues));
+            else
+              docCopy.addField(expressionKey,(Reader[])allValues);
+          } else {
+            allValues = new String[totalSize];
+            // Convert to strings throughout
+            for (IDataSource dataSource : dataSources) {
+              if (dataSource != null) {
+                for (Object o : dataSource.getStringForm()) {
+                  allValues[k++] = o;
+                }
+              }
+            }
+            if (sp.filterEmpty())
+              docCopy.addField(expressionKey,removeEmpties((String[])allValues));
+            else
+              docCopy.addField(expressionKey,(String[])allValues);
+          }
+        }
+      }
+      
+      // Finally, send the modified repository document onward to the next pipeline stage.
+      // If we'd done anything to the stream, we'd have needed to create a new RepositoryDocument object and copied the
+      // data into it, and closed the new stream after sendDocument() was called.
+      return activities.sendDocument(documentURI,docCopy);
+
+    } finally {
+      fdf.close();
     }
+  }
 
-    Iterator<String> keys = sp.getParameterKeys();
-    while (keys.hasNext())
-    {
-      String key = keys.next();
-      docCopy.addField(key,sp.getParameterValues(key));
+  protected static boolean allDates(IDataSource[] dataSources)
+    throws IOException, ManifoldCFException {
+    for (IDataSource ds : dataSources) {
+      if (ds != null && !(ds.getRawForm() instanceof Date[]))
+        return false;
     }
-    // Finally, send the modified repository document onward to the next pipeline stage.
-    // If we'd done anything to the stream, we'd have needed to create a new RepositoryDocument object and copied the
-    // data into it, and closed the new stream after sendDocument() was called.
-    return activities.sendDocument(documentURI,docCopy);
+    return true;
   }
 
-  protected static void moveData(RepositoryDocument docCopy, String target, RepositoryDocument document, String field, boolean filterEmpty)
+  protected static boolean allReaders(IDataSource[] dataSources)
+    throws IOException, ManifoldCFException {
+    for (IDataSource ds : dataSources) {
+      if (ds != null && !(ds.getRawForm() instanceof Reader[]))
+        return false;
+    }
+    return true;
+  }
+  
+  protected static void moveData(RepositoryDocument docCopy, String target, FieldDataFactory document, String field, boolean filterEmpty)
     throws ManifoldCFException, IOException
   {
     Object[] fieldData = document.getField(field);
@@ -240,9 +334,7 @@ public class ForcedMetadataConnector ext
   {
     // Output specification header
     
-    // Add Forced Metadata to tab array
-    tabsArray.add(Messages.getString(locale, "ForcedMetadata.FieldMappingTabName"));
-    tabsArray.add(Messages.getString(locale, "ForcedMetadata.ForcedMetadata"));
+    tabsArray.add(Messages.getString(locale, "ForcedMetadata.Expressions"));
 
     Map<String, Object> paramMap = new HashMap<String, Object>();
     paramMap.put("SEQNUM",Integer.toString(connectionSequenceNumber));
@@ -272,11 +364,9 @@ public class ForcedMetadataConnector ext
     paramMap.put("SEQNUM",Integer.toString(connectionSequenceNumber));
     paramMap.put("SELECTEDNUM",Integer.toString(actualSequenceNumber));
 
-    fillInForcedMetadataTab(paramMap, os);
-    fillInFieldMappingSpecificationMap(paramMap, os);
+    fillInExpressionsTab(paramMap, os);
 
-    Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_FORCED_METADATA,paramMap);
-    Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_FIELDMAPPING,paramMap);
+    Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_EXPRESSIONS,paramMap);
   }
   
   /** Process a specification post.
@@ -296,96 +386,53 @@ public class ForcedMetadataConnector ext
   {
     // Process specification post
     String seqPrefix = "s"+connectionSequenceNumber+"_";
-    String forcedCount = variableContext.getParameter(seqPrefix+"forcedmetadata_count");
-    if (forcedCount != null)
+    String expressionCount = variableContext.getParameter(seqPrefix+"expression_count");
+    if (expressionCount != null)
     {
-      int count = Integer.parseInt(forcedCount);
-      // Delete old spec data
+      int count = Integer.parseInt(expressionCount);
+      // Delete old spec data, including legacy node types we no longer use
       int i = 0;
       while (i < os.getChildCount())
       {
         SpecificationNode cn = os.getChild(i);
-        if (cn.getType().equals(NODE_PAIR))
+        if (cn.getType().equals(NODE_EXPRESSION) || cn.getType().equals(NODE_PAIR) || cn.getType().equals(NODE_FIELDMAP))
           os.removeChild(i);
         else
           i++;
       }
+
       // Now, go through form data
       for (int j = 0; j < count; j++)
       {
-        String op = variableContext.getParameter(seqPrefix+"forcedmetadata_"+j+"_op");
+        String op = variableContext.getParameter(seqPrefix+"expression_"+j+"_op");
         if (op != null && op.equals("Delete"))
           continue;
-        String paramName = variableContext.getParameter(seqPrefix+"forcedmetadata_"+j+"_name");
-        String paramValue = variableContext.getParameter(seqPrefix+"forcedmetadata_"+j+"_value");
-        SpecificationNode sn = new SpecificationNode(NODE_PAIR);
+        String paramName = variableContext.getParameter(seqPrefix+"expression_"+j+"_name");
+        String paramRemove = variableContext.getParameter(seqPrefix+"expression_"+j+"_remove");
+        String paramValue = variableContext.getParameter(seqPrefix+"expression_"+j+"_value");
+        SpecificationNode sn = new SpecificationNode(NODE_EXPRESSION);
         sn.setAttribute(ATTRIBUTE_PARAMETER,paramName);
-        sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
+        if (!(paramRemove != null && paramRemove.equals("true")))
+          sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
         os.addChild(os.getChildCount(),sn);
       }
       // Look for add operation
-      String addOp = variableContext.getParameter(seqPrefix+"forcedmetadata_op");
+      String addOp = variableContext.getParameter(seqPrefix+"expression_op");
       if (addOp != null && addOp.equals("Add"))
       {
-        String paramName = variableContext.getParameter(seqPrefix+"forcedmetadata_name");
-        String paramValue = variableContext.getParameter(seqPrefix+"forcedmetadata_value");
-        SpecificationNode sn = new SpecificationNode(NODE_PAIR);
+        String paramName = variableContext.getParameter(seqPrefix+"expression_name");
+        String paramRemove = variableContext.getParameter(seqPrefix+"expression_remove");
+        String paramValue = variableContext.getParameter(seqPrefix+"expression_value");
+        SpecificationNode sn = new SpecificationNode(NODE_EXPRESSION);
         sn.setAttribute(ATTRIBUTE_PARAMETER,paramName);
-        sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
+        if (!(paramRemove != null && paramRemove.equals("true")))
+          sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
         os.addChild(os.getChildCount(),sn);
       }
+
     }
     
-    String x = variableContext.getParameter(seqPrefix+"fieldmapping_count");
-    if (x != null && x.length() > 0)
-    {
-      // About to gather the fieldmapping nodes, so get rid of the old ones.
-      int i = 0;
-      while (i < os.getChildCount())
-      {
-        SpecificationNode node = os.getChild(i);
-        if (node.getType().equals(NODE_FIELDMAP) || node.getType().equals(NODE_KEEPMETADATA))
-          os.removeChild(i);
-        else
-          i++;
-      }
-      int count = Integer.parseInt(x);
-      i = 0;
-      while (i < count)
-      {
-        String prefix = seqPrefix+"fieldmapping_";
-        String suffix = "_"+Integer.toString(i);
-        String op = variableContext.getParameter(prefix+"op"+suffix);
-        if (op == null || !op.equals("Delete"))
-        {
-          // Gather the fieldmap etc.
-          String source = variableContext.getParameter(prefix+"source"+suffix);
-          String target = variableContext.getParameter(prefix+"target"+suffix);
-          if (target == null)
-            target = "";
-          SpecificationNode node = new SpecificationNode(NODE_FIELDMAP);
-          node.setAttribute(ATTRIBUTE_SOURCE,source);
-          node.setAttribute(ATTRIBUTE_TARGET,target);
-          os.addChild(os.getChildCount(),node);
-        }
-        i++;
-      }
-      
-      String addop = variableContext.getParameter(seqPrefix+"fieldmapping_op");
-      if (addop != null && addop.equals("Add"))
-      {
-        String source = variableContext.getParameter(seqPrefix+"fieldmapping_source");
-        String target = variableContext.getParameter(seqPrefix+"fieldmapping_target");
-        if (target == null)
-          target = "";
-        SpecificationNode node = new SpecificationNode(NODE_FIELDMAP);
-        node.setAttribute(ATTRIBUTE_SOURCE,source);
-        node.setAttribute(ATTRIBUTE_TARGET,target);
-        os.addChild(os.getChildCount(),node);
-      }
-    }
-    
-    x = variableContext.getParameter(seqPrefix+"keepallmetadata_present");
+    String x = variableContext.getParameter(seqPrefix+"keepallmetadata_present");
     if (x != null && x.length() > 0)
     {
       String keepAll = variableContext.getParameter(seqPrefix+"keepallmetadata");
@@ -454,16 +501,17 @@ public class ForcedMetadataConnector ext
     paramMap.put("SEQNUM",Integer.toString(connectionSequenceNumber));
     
     // Fill in the map with data from all tabs
-    fillInForcedMetadataTab(paramMap, os);
-    fillInFieldMappingSpecificationMap(paramMap, os);
+    fillInExpressionsTab(paramMap, os);
 
     Messages.outputResourceWithVelocity(out,locale,VIEW_SPEC,paramMap);
   }
 
-  protected static void fillInFieldMappingSpecificationMap(Map<String,Object> paramMap, Specification os)
+  protected static void fillInExpressionsTab(Map<String,Object> paramMap, Specification os)
   {
-    // Prep for field mappings
-    List<Map<String,String>> fieldMappings = new ArrayList<Map<String,String>>();
+    final Map<String,Set<String>> expressions = new HashMap<String,Set<String>>();
+    final Map<String,Set<String>> expressionAdditions = new HashMap<String,Set<String>>();
+    final Map<String,Set<String>> additions = new HashMap<String,Set<String>>();
+    
     String keepAllMetadataValue = "true";
     String filterEmptyValue = "false";
     for (int i = 0; i < os.getChildCount(); i++)
@@ -473,18 +521,44 @@ public class ForcedMetadataConnector ext
         String source = sn.getAttributeValue(ATTRIBUTE_SOURCE);
         String target = sn.getAttributeValue(ATTRIBUTE_TARGET);
         String targetDisplay;
-        if (target == null)
+        
+        expressions.put(source,new HashSet<String>());
+        if (target != null)
         {
-          target = "";
-          targetDisplay = "(remove)";
+          Set<String> sources = new HashSet<String>();
+          sources.add("${"+source+"}");
+          expressions.put(target,sources);
+        }
+      }
+      else if (sn.getType().equals(NODE_PAIR))
+      {
+        String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
+        String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+        // Since the same target is completely superceded by a NODE_PAIR, but NODE_PAIRs
+        // are cumulative, I have to build these completely and then post-process them.
+        Set<String> addition = additions.get(parameter);
+        if (addition == null)
+        {
+          addition = new HashSet<String>();
+          additions.put(parameter,addition);
+        }
+        addition.add(expressionEscape(value));
+      }
+      else if (sn.getType().equals(NODE_EXPRESSION))
+      {
+        String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
+        String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+        if (value == null) {
+          expressionAdditions.put(parameter,new HashSet<String>());
+        } else {
+          Set<String> expressionAddition = expressionAdditions.get(parameter);
+          if (expressionAddition == null)
+          {
+            expressionAddition = new HashSet<String>();
+            expressionAdditions.put(parameter,expressionAddition);
+          }
+          expressionAddition.add(value);
         }
-        else
-          targetDisplay = target;
-        Map<String,String> fieldMapping = new HashMap<String,String>();
-        fieldMapping.put("SOURCE",source);
-        fieldMapping.put("TARGET",target);
-        fieldMapping.put("TARGETDISPLAY",targetDisplay);
-        fieldMappings.add(fieldMapping);
       }
       else if (sn.getType().equals(NODE_KEEPMETADATA))
       {
@@ -495,77 +569,193 @@ public class ForcedMetadataConnector ext
         filterEmptyValue = sn.getAttributeValue(ATTRIBUTE_VALUE);
       }
     }
-    paramMap.put("FIELDMAPPINGS",fieldMappings);
-    paramMap.put("KEEPALLMETADATA",keepAllMetadataValue);
-    paramMap.put("FILTEREMPTY",filterEmptyValue);
-  }
-
-  protected static void fillInForcedMetadataTab(Map<String,Object> paramMap, Specification os)
-  {
-    // First, sort everything
-    Map<String,Set<String>> params = new HashMap<String,Set<String>>();
-    for (int i = 0; i < os.getChildCount(); i++)
+    
+    // Postprocessing.
+    // Override the moves with the additions
+    for (String parameter : additions.keySet())
     {
-      SpecificationNode sn = os.getChild(i);
-      if (sn.getType().equals(NODE_PAIR))
-      {
-        String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
-        String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
-        Set<String> values = params.get(parameter);
-        if (values == null)
-        {
-          values = new HashSet<String>();
-          params.put(parameter,values);
-        }
-        values.add(value);
-      }
+      expressions.put(parameter,additions.get(parameter));
     }
-    
-    // We construct a list of maps from the parameters
-    List<Map<String,String>> pObject = new ArrayList<Map<String,String>>();
 
-    String[] keys = new String[params.size()];
-    int j = 0;
-    for (String key : params.keySet())
+    // Override all with expression additions
+    for (String parameter : expressionAdditions.keySet())
     {
-      keys[j++] = key;
+      expressions.put(parameter,expressionAdditions.get(parameter));
     }
+
+    // Problem: how to display case where we want a null source??
+    // A: Special value
+    List<Map<String,String>> pObject = new ArrayList<Map<String,String>>();
+    String[] keys = expressions.keySet().toArray(new String[0]);
     java.util.Arrays.sort(keys);
     
     // Now, build map
     for (String key : keys)
     {
-      Set<String> values = params.get(key);
-      String[] valueArray = new String[values.size()];
-      j = 0;
-      for (String value : values)
-      {
-        valueArray[j++] = value;
-      }
-      java.util.Arrays.sort(valueArray);
-      
-      for (String value : valueArray)
-      {
+      Set<String> values = expressions.get(key);
+      if (values.size() == 0) {
         Map<String,String> record = new HashMap<String,String>();
         record.put("parameter",key);
-        record.put("value",value);
+        record.put("value","");
+        record.put("isnull","true");
         pObject.add(record);
+      } else {
+        String[] valueArray = values.toArray(new String[0]);
+        java.util.Arrays.sort(valueArray);
+        
+        for (String value : valueArray)
+        {
+          Map<String,String> record = new HashMap<String,String>();
+          record.put("parameter",key);
+          record.put("value",value);
+          record.put("isnull","false");
+          pObject.add(record);
+        }
       }
     }
     
-    paramMap.put("PARAMETERS",pObject);
+    paramMap.put("EXPRESSIONS",pObject);
+    paramMap.put("KEEPALLMETADATA",keepAllMetadataValue);
+    paramMap.put("FILTEREMPTY",filterEmptyValue);
+  }
+  
+  protected static String expressionEscape(String input) {
+    // Not doing any escaping yet
+    return input;
   }
 
+  protected static String expressionUnescape(String input) {
+    // Not doing any escaping yet
+    return input;
+  }
+  
+  public interface IDataSource {
+    public int getSize() throws IOException, ManifoldCFException;
+    public Object[] getRawForm() throws IOException, ManifoldCFException;
+    public String[] getStringForm() throws IOException, ManifoldCFException;
+  }
+  
+  protected static class StringSource implements IDataSource {
+    protected final String[] data;
+    
+    public StringSource(String data) {
+      this.data = new String[]{data};
+    }
+    
+    public StringSource(String[] data) {
+      this.data = data;
+    }
+    
+    @Override
+    public int getSize() {
+      return data.length;
+    }
+    
+    @Override
+    public Object[] getRawForm() {
+      return data;
+    }
+    
+    @Override
+    public String[] getStringForm() {
+      return data;
+    }
+  }
+  
+  protected static class FieldSource implements IDataSource {
+    
+    protected final FieldDataFactory rd;
+    protected final String fieldName;
+    
+    public FieldSource(FieldDataFactory rd, String fieldName) {
+      this.rd = rd;
+      this.fieldName = fieldName;
+    }
+    
+    @Override
+    public int getSize()
+      throws IOException, ManifoldCFException {
+      Object[] rawForm = getRawForm();
+      if (rawForm == null)
+        return 0;
+      return rawForm.length;
+    }
+    
+    @Override
+    public Object[] getRawForm()
+      throws IOException, ManifoldCFException {
+      return rd.getField(fieldName);
+    }
+    
+    @Override
+    public String[] getStringForm()
+      throws IOException, ManifoldCFException {
+      return rd.getFieldAsStrings(fieldName);
+    }
+  }
+  
+  protected static IDataSource append(IDataSource currentValues, IDataSource data)
+    throws IOException, ManifoldCFException {
+    // currentValues and data can either be:
+    // Date[], String[], or Reader[].
+    // We want to preserve the type in as high a form as possible when we compute the combinations.
+    if (currentValues == null)
+      return data;
+    if (currentValues.getSize() == 0)
+      return currentValues;
+    // Any combination causes conversion to a string, so if we get here, we can read the inputs all
+    // as strings safely.
+    String[] currentStrings = currentValues.getStringForm();
+    String[] dataStrings = data.getStringForm();
+    String[] rval = new String[currentStrings.length * dataStrings.length];
+    int rvalIndex = 0;
+    for (String currentString : currentStrings) {
+      for (String dataString : dataStrings) {
+        rval[rvalIndex++] = currentString + dataString;
+      }
+    }
+    return new StringSource(rval);
+  }
+  
+  public static IDataSource processExpression(String expression, FieldDataFactory sourceDocument)
+    throws IOException, ManifoldCFException {
+    int index = 0;
+    IDataSource input = null;
+    while (true) {
+      // If we're at the end, return the input
+      if (index == expression.length())
+        return input;
+      // Look for next field specification
+      int field = expression.indexOf("${",index);
+      if (field == -1)
+        return append(input, new StringSource(expressionUnescape(expression.substring(index))));
+      if (field > 0)
+        input = append(input, new StringSource(expressionUnescape(expression.substring(index,field))));
+      // Get the field name
+      int fieldEnd = expression.indexOf("}",field);
+      String fieldName;
+      if (fieldEnd == -1) {
+        fieldName = expression.substring(field+2);
+        return append(input, new FieldSource(sourceDocument, fieldName));
+      } else {
+        fieldName = expression.substring(field+2,fieldEnd);
+        input = append(input, new FieldSource(sourceDocument, fieldName));
+        index = fieldEnd+1;
+      }
+    }
+  }
+  
   protected static class SpecPacker {
     
-    private final Map<String,String> sourceTargets = new HashMap<String,String>();
     private final boolean keepAllMetadata;
     private final boolean filterEmpty;
-    private final Map<String,Set<String>> parameters = new HashMap<String,Set<String>>();
+    private final Map<String,Set<String>> expressions = new HashMap<String,Set<String>>();
 
     public SpecPacker(Specification os) {
       boolean keepAllMetadata = true;
       boolean filterEmpty = false;
+      final Map<String,Set<String>> additions = new HashMap<String,Set<String>>();
+      final Map<String,Set<String>> expressionAdditions = new HashMap<String,Set<String>>();
       for (int i = 0; i < os.getChildCount(); i++) {
         SpecificationNode sn = os.getChild(i);
         
@@ -579,24 +769,58 @@ public class ForcedMetadataConnector ext
           String source = sn.getAttributeValue(ATTRIBUTE_SOURCE);
           String target = sn.getAttributeValue(ATTRIBUTE_TARGET);
           
-          if (target == null) {
-            target = "";
+          expressions.put(source,new HashSet<String>());
+          // Null target means to remove the *source* from the document.
+          if (target != null) {
+            Set<String> sources = new HashSet<String>();
+            sources.add("${"+source+"}");
+            expressions.put(target,sources);
           }
-          sourceTargets.put(source, target);
         }
         else if (sn.getType().equals(NODE_PAIR))
         {
           String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
           String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
-          Set<String> params = parameters.get(parameter);
-          if (params == null)
+          // Since the same target is completely superceded by a NODE_PAIR, but NODE_PAIRs
+          // are cumulative, I have to build these completely and then post-process them.
+          Set<String> addition = additions.get(parameter);
+          if (addition == null)
           {
-            params = new HashSet<String>();
-            parameters.put(parameter,params);
+            addition = new HashSet<String>();
+            additions.put(parameter,addition);
           }
-          params.add(value);
+          addition.add(expressionEscape(value));
         }
+        else if (sn.getType().equals(NODE_EXPRESSION))
+        {
+          String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
+          String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+          if (value == null) {
+            expressionAdditions.put(parameter,new HashSet<String>());
+          } else {
+            Set<String> expressionAddition = expressionAdditions.get(parameter);
+            if (expressionAddition == null)
+            {
+              expressionAddition = new HashSet<String>();
+              expressionAdditions.put(parameter,expressionAddition);
+            }
+            expressionAddition.add(value);
+          }
+        }
+      }
+      
+      // Override the moves with the additions
+      for (String parameter : additions.keySet())
+      {
+        expressions.put(parameter,additions.get(parameter));
+      }
+
+      // Override all with expression additions
+      for (String parameter : expressionAdditions.keySet())
+      {
+        expressions.put(parameter,expressionAdditions.get(parameter));
       }
+      
       this.keepAllMetadata = keepAllMetadata;
       this.filterEmpty = filterEmpty;
     }
@@ -605,25 +829,16 @@ public class ForcedMetadataConnector ext
       StringBuilder sb = new StringBuilder();
       int i;
       
-      // Mappings
-      final String[] sortArray = new String[sourceTargets.size()];
-      i = 0;
-      for (String source : sourceTargets.keySet()) {
-        sortArray[i++] = source;
-      }
+      final String[] sortArray = expressions.keySet().toArray(new String[0]);
       java.util.Arrays.sort(sortArray);
-      
-      List<String> packedMappings = new ArrayList<String>();
-      String[] fixedList = new String[2];
-      for (String source : sortArray) {
-        String target = sourceTargets.get(source);
-        StringBuilder localBuffer = new StringBuilder();
-        fixedList[0] = source;
-        fixedList[1] = target;
-        packFixedList(localBuffer,fixedList,':');
-        packedMappings.add(localBuffer.toString());
+      // Pack the list of keys
+      packList(sb,sortArray,'+');
+      for (String key : sortArray) {
+        Set<String> values = expressions.get(key);
+        String[] valueArray = values.toArray(new String[0]);
+        java.util.Arrays.sort(valueArray);
+        packList(sb,valueArray,'+');
       }
-      packList(sb,packedMappings,'+');
 
       // Keep all metadata
       if (keepAllMetadata)
@@ -631,30 +846,6 @@ public class ForcedMetadataConnector ext
       else
         sb.append('-');
       
-      // Get the keys and sort them
-      final String[] keys = new String[parameters.size()];
-      int j = 0;
-      for (String key : parameters.keySet())
-      {
-        keys[j++] = key;
-      }
-      java.util.Arrays.sort(keys);
-      // Pack the list of keys
-      packList(sb,keys,'+');
-      // Now, go through each key and individually pack the values
-      for (String key : keys)
-      {
-        Set<String> values = parameters.get(key);
-        String[] valueArray = new String[values.size()];
-        j = 0;
-        for (String value : values)
-        {
-          valueArray[j++] = value;
-        }
-        java.util.Arrays.sort(valueArray);
-        packList(sb,valueArray,'+');
-      }
-
       // Filter empty
       if (filterEmpty)
         sb.append('+');
@@ -664,8 +855,14 @@ public class ForcedMetadataConnector ext
       return sb.toString();
     }
     
-    public String getMapping(String source) {
-      return sourceTargets.get(source);
+    public Iterator<String> getExpressionKeys()
+    {
+      return expressions.keySet().iterator();
+    }
+    
+    public Set<String> getExpressionValues(String key)
+    {
+      return expressions.get(key);
     }
     
     public boolean keepAllMetadata() {
@@ -675,27 +872,119 @@ public class ForcedMetadataConnector ext
     public boolean filterEmpty() {
       return filterEmpty;
     }
+  }
+  
+  /** This class provides unique Reader and other field instances, when requested, based
+  * on an input RepositoryDocument.  It does this by pulling the values of the field into
+  * a CharacterInput implementation, thus making a temporary file copy.  So it is imperative
+  * that this object is closed when it is no longer needed.
+  */
+  public static class FieldDataFactory {
     
-    public Iterator<String> getParameterKeys()
-    {
-      return parameters.keySet().iterator();
+    protected final RepositoryDocument sourceDocument;
+    
+    // Readers (organized by metadata)
+    protected final Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
+
+    public FieldDataFactory(RepositoryDocument sourceDocument) {
+      this.sourceDocument = sourceDocument;
     }
     
-    public String[] getParameterValues(String key)
+    public void close()
+      throws ManifoldCFException
     {
-      Set<String> values = parameters.get(key);
-      if (values == null)
-        return null;
-      String[] rval = new String[values.size()];
-      int i = 0;
-      for (String value : values)
+      for (String key : metadataReaders.keySet())
+      {
+        CharacterInput[] rt = metadataReaders.get(key);
+        for (CharacterInput r : rt)
+        {
+          r.discard();
+        }
+      }
+    }
+    
+    public Object[] getField(String fieldName)
+      throws IOException, ManifoldCFException {
+      CharacterInput[] inputs = metadataReaders.get(fieldName);
+      if (inputs == null) {
+        // Either never seen the field before, or it's not a Reader
+        Object[] fieldValues = sourceDocument.getField(fieldName);
+        if (fieldValues == null)
+          return fieldValues;
+        if (fieldValues instanceof Reader[]) {
+          // Create a copy
+          CharacterInput[] newValues = new CharacterInput[fieldValues.length];
+          try {
+            // Populate newValues
+            for (int i = 0; i < newValues.length; i++)
+            {
+              newValues[i] = new TempFileCharacterInput((Reader)fieldValues[i]);
+            }
+            metadataReaders.put(fieldName,newValues);
+            inputs = newValues;
+          } catch (Throwable e) {
+            for (CharacterInput r : newValues)
+            {
+              if (r != null)
+                r.discard();
+            }
+            if (e instanceof IOException)
+              throw (IOException)e;
+            else if (e instanceof RuntimeException)
+              throw (RuntimeException)e;
+            else if (e instanceof Error)
+              throw (Error)e;
+            else
+              throw new RuntimeException("Unknown exception type: "+e.getClass().getName()+": "+e.getMessage(),e);
+          }
+        } else {
+          return fieldValues;
+        }
+      }
+        
+      Reader[] newReaders = new Reader[inputs.length];
+      for (int i = 0; i < inputs.length; i++)
       {
-        rval[i++] = value;
+        inputs[i].doneWithStream();
+        newReaders[i] = inputs[i].getStream();
+      }
+      return newReaders;
+    }
+    
+    public String[] getFieldAsStrings(String fieldName)
+      throws IOException, ManifoldCFException {
+      CharacterInput[] cilist = metadataReaders.get(fieldName);
+      if (cilist == null)
+        return sourceDocument.getFieldAsStrings(fieldName);
+      
+      // We've created a local array of CharacterInputs from this field.  We'll need to convert these
+      // to strings.
+      char[] buffer = new char[65536];
+      String[] rval = new String[cilist.length];
+      for (int i = 0; i < rval.length; i++) {
+        CharacterInput ci = cilist[i];
+        ci.doneWithStream();
+        Reader r = ci.getStream();
+        // Read into a buffer
+        StringBuilder newValue = new StringBuilder();
+        while (true)
+        {
+          int amt = r.read(buffer);
+          if (amt == -1)
+            break;
+          newValue.append(buffer,0,amt);
+        }
+        rval[i] = newValue.toString();
+      }
+      sourceDocument.addField(fieldName,rval);
+      metadataReaders.remove(fieldName);
+      for (CharacterInput ci : cilist) {
+        ci.discard();
       }
       return rval;
     }
   }
-
+  
 }
 
 

Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties Mon Jan  5 18:40:18 2015
@@ -13,26 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ForcedMetadata.ForcedMetadata=Add metadata
-ForcedMetadata.FieldMappingTabName=Move metadata
-
-ForcedMetadata.ForcedMetadataNameMustNotBeNull=Added metadata name must not be null
-ForcedMetadata.ForcedMetadataColon=Added metadata:
+ForcedMetadata.Expressions=Metadata expressions
+ForcedMetadata.RemoveQ=Remove this parameter?
+ForcedMetadata.ParameterValueExpression=Expression ("${fieldname}" references a field)
+ForcedMetadata.MetadataExpressionsColon=Metadata expressions:
+ForcedMetadata.NoExpressionsSpecified=No expressions provided
+ForcedMetadata.Deleteexpressionnumber=Delete expression #
+ForcedMetadata.Addexpression=Add expression
+ForcedMetadata.True=true
+ForcedMetadata.False=false
+ForcedMetadata.ExpressionMetadataNameMustNotBeNull=Expression metadata name must not be null
 ForcedMetadata.ParameterName=Parameter name
-ForcedMetadata.ParameterValue=Parameter value
 ForcedMetadata.Delete=Delete
-ForcedMetadata.Deleteforcedmetadatanumber=Delete added metadata #
 ForcedMetadata.Add=Add
-ForcedMetadata.Addforcedmetadata=Add metadata item
-ForcedMetadata.NoForcedMetadataSpecified=No added metadata specified
-ForcedMetadata.FieldMappings=Metadata mappings:
-ForcedMetadata.MetadataFieldName=Incoming metadata name
-ForcedMetadata.FinalFieldName=Final metadata name
-ForcedMetadata.NoFieldMappingSpecified=No metadata mapping specified
 ForcedMetadata.KeepAllMetadata=Keep all incoming metadata
 ForcedMetadata.FilterEmpty=Remove empty metadata values
-ForcedMetadata.Add=Add
-ForcedMetadata.AddFieldMapping=Add metadata mapping
-ForcedMetadata.Delete=Delete
-ForcedMetadata.DeleteFieldMapping=Delete metadata mapping #
 ForcedMetadata.NoFieldNameSpecified=Please specify a metadata name

Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties Mon Jan  5 18:40:18 2015
@@ -13,26 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ForcedMetadata.ForcedMetadata=強制メタデータ
-ForcedMetadata.FieldMappingTabName=フィールドマッピング
-
-ForcedMetadata.ForcedMetadataNameMustNotBeNull=強制メタデータ名を必ず入力してください
-ForcedMetadata.ForcedMetadataColon=強制メタデータ:
+ForcedMetadata.Expressions=Metadata expressions
+ForcedMetadata.RemoveQ=Remove this parameter?
+ForcedMetadata.ParameterValueExpression=Expression ("${fieldname}" references a field)
+ForcedMetadata.MetadataExpressionsColon=Metadata expressions:
+ForcedMetadata.NoExpressionsSpecified=No expressions provided
+ForcedMetadata.Deleteexpressionnumber=Delete expression #
+ForcedMetadata.Addexpression=Add expression
+ForcedMetadata.True=true
+ForcedMetadata.False=false
+ForcedMetadata.ExpressionMetadataNameMustNotBeNull=Expression metadata name must not be null
 ForcedMetadata.ParameterName=パラメータ名
-ForcedMetadata.ParameterValue=パラメータ値
 ForcedMetadata.Delete=削除
-ForcedMetadata.Deleteforcedmetadatanumber=強制メタデータを削除 #
 ForcedMetadata.Add=追加
-ForcedMetadata.Addforcedmetadata=強制メタデータを追加してください
-ForcedMetadata.NoForcedMetadataSpecified=強制メタデータを指定してください
-ForcedMetadata.FieldMappings=フィールドマッピング:
-ForcedMetadata.MetadataFieldName=メタデータフィールド名
-ForcedMetadata.FinalFieldName=最後のフィールド名
-ForcedMetadata.NoFieldMappingSpecified=フィールドマッピングを指定してください
 ForcedMetadata.KeepAllMetadata=全メタデータを保存
 ForcedMetadata.FilterEmpty=Remove empty metadata values
-ForcedMetadata.Add=追加
-ForcedMetadata.AddFieldMapping=フィールドマッピングを追加
-ForcedMetadata.Delete=削除
-ForcedMetadata.DeleteFieldMapping=フィールドマッピングを削除 #
 ForcedMetadata.NoFieldNameSpecified=フィールド名を入力してください

Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties Mon Jan  5 18:40:18 2015
@@ -13,26 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ForcedMetadata.ForcedMetadata=强制性元数据
-ForcedMetadata.FieldMappingTabName=字段映射
-
-ForcedMetadata.ForcedMetadataNameMustNotBeNull=强制性元数据名不能为空
-ForcedMetadata.ForcedMetadataColon=强制性元数据:
+ForcedMetadata.Expressions=Metadata expressions
+ForcedMetadata.RemoveQ=Remove this parameter?
+ForcedMetadata.ParameterValueExpression=Expression ("${fieldname}" references a field)
+ForcedMetadata.MetadataExpressionsColon=Metadata expressions:
+ForcedMetadata.NoExpressionsSpecified=No expressions provided
+ForcedMetadata.Deleteexpressionnumber=Delete expression #
+ForcedMetadata.Addexpression=Add expression
+ForcedMetadata.True=true
+ForcedMetadata.False=false
+ForcedMetadata.ExpressionMetadataNameMustNotBeNull=Expression metadata name must not be null
 ForcedMetadata.ParameterName=参数名
-ForcedMetadata.ParameterValue=参数值
 ForcedMetadata.Delete=删除
-ForcedMetadata.Deleteforcedmetadatanumber=删除强制性元数据 #
 ForcedMetadata.Add=添加
-ForcedMetadata.Addforcedmetadata=添加强制性元数据
-ForcedMetadata.NoForcedMetadataSpecified=未指定强制性元数据
-ForcedMetadata.FieldMappings=字段映射:
-ForcedMetadata.MetadataFieldName=元数据字段名
-ForcedMetadata.FinalFieldName=最终字段名
-ForcedMetadata.NoFieldMappingSpecified=未指定字段映射
 ForcedMetadata.KeepAllMetadata=保持所有元数据
 ForcedMetadata.FilterEmpty=Remove empty metadata values
-ForcedMetadata.Add=添加
-ForcedMetadata.AddFieldMapping=添加字段映射
-ForcedMetadata.Delete=删除
-ForcedMetadata.DeleteFieldMapping=删除字段映射 #
 ForcedMetadata.NoFieldNameSpecified=请指定字段名

Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js Mon Jan  5 18:40:18 2015
@@ -18,6 +18,27 @@
 <script type="text/javascript">
 <!--
 
+function s${SEQNUM}_AddExpression()
+{
+  if (editjob.s${SEQNUM}_expression_name.value == "")
+  {
+    alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('ForcedMetadata.ExpressionMetadataNameMustNotBeNull'))");
+    editjob.s${SEQNUM}_expression_name.focus();
+    return;
+  }
+  document.editjob.s${SEQNUM}_expression_op.value="Add";
+  postFormSetAnchor("s${SEQNUM}_expression_tag");
+}
+	
+function s${SEQNUM}_DeleteExpression(n)
+{
+  eval("document.editjob.s${SEQNUM}_expression_"+n+"_op.value = 'Delete'");
+  if (n == 0)
+    postFormSetAnchor("s${SEQNUM}_expression_tag");
+  else
+    postFormSetAnchor("s${SEQNUM}_expression_"+(n-1)+"_tag");
+}
+
 function s${SEQNUM}_AddForcedMetadata()
 {
   if (editjob.s${SEQNUM}_forcedmetadata_name.value == "")

Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html Mon Jan  5 18:40:18 2015
@@ -17,58 +17,16 @@
 
 <table class="displaytable">
   <tr>
-    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FieldMappings'))</nobr></td>
-    <td class="boxcell">
-      <table class="formtable">
-        <tr class="formheaderrow">
-          <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.MetadataFieldName'))</nobr></td>
-          <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FinalFieldName'))</nobr></td>
-        </tr>
-#set($fieldcounter = 0)
-#foreach($fieldmapping in $FIELDMAPPINGS)
-  #if(($fieldcounter % 2) == 0)
-        <tr class="evenformrow">
-  #else
-        <tr class="oddformrow">
-  #end
-          <td class="formcolumncell">
-            <nobr>$Encoder.bodyEscape($fieldmapping.get('SOURCE'))</nobr>
-          </td>
-          <td class="formcolumncell">
-            <nobr>$Encoder.bodyEscape($fieldmapping.get('TARGETDISPLAY'))</nobr>
-          </td>
-        </tr>
-  #set($fieldcounter = $fieldcounter + 1)
-#end
-#if($fieldcounter == 0)
-        <tr class="formrow"><td class="formmessage" colspan="2">$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.NoFieldMappingSpecified'))</td></tr>
-#end
-      </table>
-    </td>
-  </tr>
-  <tr><td class="separator" colspan="2"><hr/></td></tr>
-  <tr>
-    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.KeepAllMetadata'))</nobr></td>
-    <td class="value"><nobr>$Encoder.bodyEscape($KEEPALLMETADATA)</nobr></td>
-  </tr>
-  <tr>
-    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FilterEmpty'))</nobr></td>
-    <td class="value"><nobr>$Encoder.bodyEscape($FILTEREMPTY)</nobr></td>
-  </tr>
-
-  <tr>
-    <td class="separator" colspan="4"><hr/></td>
-  </tr>
-  <tr>
-    <td class="description" colspan="1"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ForcedMetadataColon'))</nobr></td>
+    <td class="description" colspan="1"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.MetadataExpressionsColon'))</nobr></td>
     <td class="boxcell" colspan="3">
       <table class="formtable">
         <tr class="formheaderrow">
           <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ParameterName'))</nobr></td>
-          <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ParameterValue'))</nobr></td>
+          <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.RemoveQ'))</nobr></td>
+          <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ParameterValueExpression'))</nobr></td>
         </tr>
   #set($paramcounter = 0)
-  #foreach($paramrecord in $PARAMETERS)
+  #foreach($paramrecord in $EXPRESSIONS)
     #if(($paramcounter % 2) == 0)
         <tr class="evenformrow">
     #else
@@ -78,15 +36,34 @@
             <nobr>$Encoder.bodyEscape($paramrecord.get('parameter'))</nobr>
           </td>
           <td class="formcolumncell">
+            <nobr>
+    #if($paramrecord.get('isnull') == 'true')
+              $Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.True'))
+    #end
+    #if($paramrecord.get('isnull') == 'false')
+              $Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.False'))
+    #end
+            </nobr>
+          </td>
+          <td class="formcolumncell">
             <nobr>$Encoder.bodyEscape($paramrecord.get('value'))</nobr>
           </td>
         </tr>
     #set($paramcounter = $paramcounter + 1)
   #end
   #if($paramcounter == 0)
-        <tr class="formrow"><td colspan="2" class="formcolumnmessage"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.NoForcedMetadataSpecified'))</nobr></td></tr>
+        <tr class="formrow"><td colspan="4" class="formcolumnmessage"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.NoExpressionsSpecified'))</nobr></td></tr>
   #end
       </table>
     </td>
   </tr>
+  <tr><td class="separator" colspan="4"><hr/></td></tr>
+  <tr>
+    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.KeepAllMetadata'))</nobr></td>
+    <td class="value"><nobr>$Encoder.bodyEscape($KEEPALLMETADATA)</nobr></td>
+  </tr>
+  <tr>
+    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FilterEmpty'))</nobr></td>
+    <td class="value"><nobr>$Encoder.bodyEscape($FILTEREMPTY)</nobr></td>
+  </tr>
 </table>

Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan  5 18:40:18 2015
@@ -113,4 +113,4 @@
 /manifoldcf/branches/CONNECTORS-989/framework:1611600-1612101
 /manifoldcf/branches/CONNECTORS-990/framework:1610284-1610707
 /manifoldcf/trunk:1629122
-/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
 641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608
+/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
 641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605

Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java Mon Jan  5 18:40:18 2015
@@ -43,7 +43,7 @@ public class RepositoryDocumentFactory
   protected BinaryInput binaryTracker;
   
   // Readers (organized by metadata)
-  protected Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
+  protected final Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
   
   /** Constructor.
   * Pass a RepositoryDocument.  This constructor reads all streams and stores them in

Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java Mon Jan  5 18:40:18 2015
@@ -415,6 +415,17 @@ public class RepositoryDocument
     return binaryLength;
   }
 
+  /** Remove a field.
+  *@param fieldName is the field name.
+  */
+  public void removeField(String fieldName)
+  {
+    fields.remove(fieldName);
+    stringFields.remove(fieldName);
+    readerFields.remove(fieldName);
+    dateFields.remove(fieldName);
+  }
+  
   /** Add/remove a multivalue date field.
   *@param fieldName is the field name.
   *@param fieldData is the multi-valued data (an array of Dates).  Null means
@@ -577,6 +588,12 @@ public class RepositoryDocument
         newValues[i] = newValue.toString();
       }
       stringFields.put(fieldName,newValues);
+      // Reader is no longer useful, since we've read it to the end.
+      // Remove it from the record accordingly.
+      // NOTE WELL: This could cause side effects if the same
+      // field is accessed simultaneously two different ways!
+      readerFields.remove(fieldName);
+      fields.put(fieldName,newValues);
       return newValues;
     }
     else



Mime
View raw message