manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1626228 [6/10] - in /manifoldcf/branches/dev_1x: ./ connectors/alfresco/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfresco/ connectors/cmis/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/cmis/ conne...
Date Fri, 19 Sep 2014 14:22:28 GMT
Modified: manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java?rev=1626228&r1=1626227&r2=1626228&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraRepositoryConnector.java Fri Sep 19 14:22:27 2014
@@ -51,6 +51,7 @@ import org.apache.manifoldcf.core.interf
 import org.apache.manifoldcf.crawler.interfaces.DocumentSpecification;
 import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
 import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
+import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
 import java.util.Map.Entry;
 
 /**
@@ -939,88 +940,97 @@ public class JiraRepositoryConnector ext
   }
   
 
-  
-  /**
-   * Process a set of documents. This is the method that should cause each
-   * document to be fetched, processed, and the results either added to the
-   * queue of documents for the current job, and/or entered into the
-   * incremental ingestion manager. The document specification allows this
-   * class to filter what is done based on the job.
-   *
-   * @param documentIdentifiers is the set of document identifiers to process.
-   * @param versions is the corresponding document versions to process, as
-   * returned by getDocumentVersions() above. The implementation may choose to
-   * ignore this parameter and always process the current version.
-   * @param activities is the interface this method should use to queue up new
-   * document references and ingest documents.
-   * @param spec is the document specification.
-   * @param scanOnly is an array corresponding to the document identifiers. It
-   * is set to true to indicate when the processing should only find other
-   * references, and should not actually call the ingestion methods.
-   * @param jobMode is an integer describing how the job is being run, whether
-   * continuous or once-only.
-   */
-  @SuppressWarnings("unchecked")
+  /** Process a set of documents.
+  * This is the method that should cause each document to be fetched, processed, and the results either added
+  * to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
+  * The document specification allows this class to filter what is done based on the job.
+  * The connector will be connected before this method can be called.
+  *@param documentIdentifiers is the set of document identifiers to process.
+  *@param statuses are the currently-stored document versions for each document in the set of document identifiers
+  * passed in above.
+  *@param activities is the interface this method should use to queue up new document references
+  * and ingest documents.
+  *@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
+  *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
+  */
   @Override
-  public void processDocuments(String[] documentIdentifiers, String[] versions,
-      IProcessActivity activities, DocumentSpecification spec,
-      boolean[] scanOnly) throws ManifoldCFException, ServiceInterruption {
+  public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec,
+    IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
+    throws ManifoldCFException, ServiceInterruption {
 
-    Logging.connectors.debug("JIRA: Inside processDocuments");
+    // Forced acls
+    String[] acls = getAcls(spec);
+    if (acls != null)
+      java.util.Arrays.sort(acls);
 
-    for (int i = 0; i < documentIdentifiers.length; i++) {
-      String nodeId = documentIdentifiers[i];
-      String version = versions[i];
-      
-      long startTime = System.currentTimeMillis();
-      String errorCode = "FAILED";
-      String errorDesc = StringUtils.EMPTY;
-      Long fileSize = null;
-      boolean doLog = false;
+    for (String documentIdentifier : documentIdentifiers) {
       
-      try {
-        if (Logging.connectors.isDebugEnabled()) {
-          Logging.connectors.debug("JIRA: Processing document identifier '"
-              + nodeId + "'");
+      if (documentIdentifier.startsWith("I-")) {
+        // It is an issue
+        String versionString;
+        String[] aclsToUse;
+        String issueID;
+        JiraIssue jiraFile;
+        
+        issueID = documentIdentifier.substring(2);
+        jiraFile = getIssue(issueID);
+        if (jiraFile == null) {
+          activities.deleteDocument(documentIdentifier);
+          continue;
         }
+        Date rev = jiraFile.getUpdatedDate();
+        if (rev != null) {
+          StringBuilder sb = new StringBuilder();
 
-        if (!scanOnly[i]) {
-          doLog = true;
+          if (acls == null) {
+            // Get acls from issue
+            List<String> users = getUsers(issueID);
+            aclsToUse = (String[])users.toArray(new String[0]);
+            java.util.Arrays.sort(aclsToUse);
+          } else {
+            aclsToUse = acls;
+          }
+          
+          // Acls
+          packList(sb,aclsToUse,'+');
+          if (aclsToUse.length > 0) {
+            sb.append('+');
+            pack(sb,defaultAuthorityDenyToken,'+');
+          } else
+            sb.append('-');
+          sb.append(rev.toString());
+          versionString = sb.toString();
+        } else {
+          //a jira document that doesn't contain versioning information will NEVER be processed.
+          // I don't know what this means, and whether it can ever occur.
+          activities.deleteDocument(documentIdentifier);
+          continue;
+        }
 
-          if (nodeId.startsWith("I-")) {
-            // It's an issue
-            String issueKey = nodeId.substring(2);
-            JiraIssue jiraFile = getIssue(issueKey);
-            if (jiraFile == null) {
-              activities.deleteDocument(nodeId);
-              continue;
-            }
-            
+        if (activities.checkDocumentNeedsReindexing(documentIdentifier,versionString))
+        {
+          
+          long startTime = System.currentTimeMillis();
+          String errorCode = "FAILED";
+          String errorDesc = StringUtils.EMPTY;
+          Long fileSize = null;
+          
+          try {
             if (Logging.connectors.isDebugEnabled()) {
-              Logging.connectors.debug("JIRA: This issue exists: " + jiraFile.getKey());
-            }
-
-            // Unpack the version string
-            ArrayList acls = new ArrayList();
-            StringBuilder denyAclBuffer = new StringBuilder();
-            int index = unpackList(acls,version,0,'+');
-            if (index < version.length() && version.charAt(index++) == '+') {
-              index = unpack(denyAclBuffer,version,index,'+');
+              Logging.connectors.debug("JIRA: Processing document identifier '"
+                  + documentIdentifier + "'");
             }
 
             //otherwise process
             RepositoryDocument rd = new RepositoryDocument();
               
             // Turn into acls and add into description
-            String[] aclArray = new String[acls.size()];
-            for (int j = 0; j < aclArray.length; j++) {
-              aclArray[j] = (String)acls.get(j);
-            }
-            rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclArray);
-            if (denyAclBuffer.length() > 0) {
-              String[] denyAclArray = new String[]{denyAclBuffer.toString()};
-              rd.setSecurityDenyACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,denyAclArray);
-            }
+            String[] denyAclsToUse;
+            if (aclsToUse.length > 0)
+              denyAclsToUse = new String[]{defaultAuthorityDenyToken};
+            else
+              denyAclsToUse = new String[0];
+            rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclsToUse,denyAclsToUse);
 
             // Now do standard stuff
               
@@ -1052,7 +1062,7 @@ public class JiraRepositoryConnector ext
               InputStream is = new ByteArrayInputStream(documentBytes);
               try {
                 rd.setBinary(is, documentBytes.length);
-                activities.ingestDocumentWithException(nodeId, version, documentURI, rd);
+                activities.ingestDocumentWithException(documentIdentifier, versionString, documentURI, rd);
                 // No errors.  Record the fact that we made it.
                 errorCode = "OK";
                 fileSize = new Long(documentBytes.length);
@@ -1062,16 +1072,20 @@ public class JiraRepositoryConnector ext
             } catch (java.io.IOException e) {
               handleIOException(e);
             }
+          } finally {
+            activities.recordActivity(new Long(startTime), ACTIVITY_READ,
+              fileSize, documentIdentifier, errorCode, errorDesc, null);
           }
+
         }
-      } finally {
-        if (doLog)
-          activities.recordActivity(new Long(startTime), ACTIVITY_READ,
-            fileSize, nodeId, errorCode, errorDesc, null);
+      } else {
+        // Unrecognized identifier type
+        activities.deleteDocument(documentIdentifier);
+        continue;
       }
     }
   }
-
+  
   protected static String getJiraBody(JiraIssue jiraFile) {
     String summary = jiraFile.getSummary();
     String description = jiraFile.getDescription();
@@ -1086,78 +1100,11 @@ public class JiraRepositoryConnector ext
     return body.toString();
   }
 
-  /**
-   * The short version of getDocumentVersions. Get document versions given an
-   * array of document identifiers. This method is called for EVERY document
-   * that is considered. It is therefore important to perform as little work
-   * as possible here.
-   *
-   * @param documentIdentifiers is the array of local document identifiers, as
-   * understood by this connector.
-   * @param spec is the current document specification for the current job. If
-   * there is a dependency on this specification, then the version string
-   * should include the pertinent data, so that reingestion will occur when
-   * the specification changes. This is primarily useful for metadata.
-   * @return the corresponding version strings, with null in the places where
-   * the document no longer exists. Empty version strings indicate that there
-   * is no versioning ability for the corresponding document, and the document
-   * will always be processed.
-   */
-  @Override
-  public String[] getDocumentVersions(String[] documentIdentifiers,
-      DocumentSpecification spec) throws ManifoldCFException,
-      ServiceInterruption {
-
-    // Forced acls
-    String[] acls = getAcls(spec);
-    if (acls != null)
-      java.util.Arrays.sort(acls);
-
-    String[] rval = new String[documentIdentifiers.length];
-    for (int i = 0; i < rval.length; i++) {
-      String nodeId = documentIdentifiers[i];
-      if (nodeId.startsWith("I-")) {
-        // It is an issue
-        String issueID = nodeId.substring(2);
-        JiraIssue jiraFile = getIssue(issueID);
-        Date rev = jiraFile.getUpdatedDate();
-        if (rev != null) {
-          StringBuilder sb = new StringBuilder();
-
-          String[] aclsToUse;
-          if (acls == null) {
-            // Get acls from issue
-            List<String> users = getUsers(issueID);
-            aclsToUse = (String[])users.toArray(new String[0]);
-            java.util.Arrays.sort(aclsToUse);
-          } else {
-            aclsToUse = acls;
-          }
-          
-          // Acls
-          packList(sb,aclsToUse,'+');
-          if (aclsToUse.length > 0) {
-            sb.append('+');
-            pack(sb,defaultAuthorityDenyToken,'+');
-          } else
-            sb.append('-');
-          sb.append(rev.toString());
-          rval[i] = sb.toString();
-        } else {
-          //a jira document that doesn't contain versioning information will NEVER be processed.
-          // I don't know what this means, and whether it can ever occur.
-          rval[i] = null;
-        }
-      }
-    }
-    return rval;
-  }
-
   /** Grab forced acl out of document specification.
   *@param spec is the document specification.
   *@return the acls, or null if security is on (and the acls need to be fetched)
   */
-  protected static String[] getAcls(DocumentSpecification spec) {
+  protected static String[] getAcls(Specification spec) {
     Set<String> map = new HashSet<String>();
     for (int i = 0; i < spec.getChildCount(); i++) {
       SpecificationNode sn = spec.getChild(i);

Modified: manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1626228&r1=1626227&r2=1626228&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Fri Sep 19 14:22:27 2014
@@ -96,11 +96,6 @@ public class LivelinkConnector extends o
   /** Deny access token for default authority */
   private final static String defaultAuthorityDenyToken = GLOBAL_DENY_TOKEN;
 
-  // Livelink does not have "deny" permissions, and there is no such thing as a document with no tokens, so it is safe to not have a local "deny" token.
-  // However, people feel that a suspenders-and-belt approach is called for, so this restriction has been added.
-  // Livelink tokens are numbers, "SYSTEM", or "GUEST", so they can't collide with the standard form.
-  private static final String denyToken = "DEAD_AUTHORITY";
-
   // A couple of very important points.
   // First, the canonical document identifier has the following form:
   // <D|F>[<volume_id>:]<object_id>
@@ -1066,152 +1061,80 @@ public class LivelinkConnector extends o
 
   }
 
-
-  /** Get document versions given an array of document identifiers.
-  * This method is called for EVERY document that is considered. It is
-  * therefore important to perform as little work as possible here.
-  *@param documentIdentifiers is the array of local document identifiers, as understood by this connector.
-  *@param oldVersions is the corresponding array of version strings that have been saved for the document identifiers.
-  *   A null value indicates that this is a first-time fetch, while an empty string indicates that the previous document
-  *   had an empty version string.
-  *@param activities is the interface this method should use to perform whatever framework actions are desired.
-  *@param spec is the current document specification for the current job.  If there is a dependency on this
-  * specification, then the version string should include the pertinent data, so that reingestion will occur
-  * when the specification changes.  This is primarily useful for metadata.
+  /** Process a set of documents.
+  * This is the method that should cause each document to be fetched, processed, and the results either added
+  * to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
+  * The document specification allows this class to filter what is done based on the job.
+  * The connector will be connected before this method can be called.
+  *@param documentIdentifiers is the set of document identifiers to process.
+  *@param statuses are the currently-stored document versions for each document in the set of document identifiers
+  * passed in above.
+  *@param activities is the interface this method should use to queue up new document references
+  * and ingest documents.
   *@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
   *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
-  *@return the corresponding version strings, with null in the places where the document no longer exists.
-  * Empty version strings indicate that there is no versioning ability for the corresponding document, and the document
-  * will always be processed.
   */
   @Override
-  public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
-    DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+  public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec,
+    IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
     throws ManifoldCFException, ServiceInterruption
   {
-    getSession();
-
+    
     // Initialize a "livelink context", to minimize the number of objects we have to fetch
     LivelinkContext llc = new LivelinkContext();
+    // Initialize the table of catid's.
+    // Keeping this around will allow us to benefit from batching of documents.
+    MetadataDescription desc = new MetadataDescription(llc);
     
     // First, process the spec to get the string we tack on
+    SystemMetadataDescription sDesc = new SystemMetadataDescription(llc,spec);
+
 
     // Read the forced acls.  A null return indicates that security is disabled!!!
     // A zero-length return indicates that the native acls should be used.
     // All of this is germane to how we ingest the document, so we need to note it in
     // the version string completely.
-    String[] acls = getAcls(spec);
+    String[] acls = sDesc.getAcls();
     // Sort it, in case it is needed.
     if (acls != null)
       java.util.Arrays.sort(acls);
 
-    // Look at the metadata attributes.
-    // So that the version strings are comparable, we will put them in an array first, and sort them.
-    String pathAttributeName = null;
-    String pathSeparator = null;
-    Set<String> holder = new HashSet<String>();
-    MatchMap matchMap = new MatchMap();
-    boolean includeAllMetadata = false;
-    int i = 0;
-    while (i < spec.getChildCount())
-    {
-      SpecificationNode n = spec.getChild(i++);
-      if (n.getType().equals("allmetadata"))
-      {
-        String isAll = n.getAttributeValue("all");
-        if (isAll != null && isAll.equals("true"))
-          includeAllMetadata = true;
-      }
-      else if (n.getType().equals("metadata"))
-      {
-        String category = n.getAttributeValue("category");
-        String attributeName = n.getAttributeValue("attribute");
-        String isAll = n.getAttributeValue("all");
-        if (isAll != null && isAll.equals("true"))
-        {
-          // Locate all metadata items for the specified category path,
-          // and enter them into the array
-          String[] attrs = getCategoryAttributes(llc,category);
-          if (attrs != null)
-          {
-            int j = 0;
-            while (j < attrs.length)
-            {
-              attributeName = attrs[j++];
-              String metadataName = packCategoryAttribute(category,attributeName);
-              holder.add(metadataName);
-            }
-          }
-        }
-        else
-        {
-          String metadataName = packCategoryAttribute(category,attributeName);
-          holder.add(metadataName);
-        }
-      }
-      else if (n.getType().equals("pathnameattribute"))
-      {
-        pathAttributeName = n.getAttributeValue("value");
-        pathSeparator = n.getAttributeValue("separator");
-        if (pathSeparator == null)
-          pathSeparator = "/";
-      }
-      else if (n.getType().equals("pathmap"))
-      {
-        // Path mapping info also needs to be looked at, because it affects what is
-        // ingested.
-        String pathMatch = n.getAttributeValue("match");
-        String pathReplace = n.getAttributeValue("replace");
-        matchMap.appendMatchPair(pathMatch,pathReplace);
-      }
-
-    }
-
     // Prepare the specified metadata
-    StringBuilder metadataString = null;
+    String metadataString = null;
+    String[] specifiedMetadataAttributes = null;
     CategoryPathAccumulator catAccum = null;
-    if (!includeAllMetadata)
+    if (!sDesc.includeAllMetadata())
     {
-      metadataString = new StringBuilder();
-      // Put into an array
-      String[] sortArray = new String[holder.size()];
-      i = 0;
-      for (String attrName : holder)
-      {
-        sortArray[i++] = attrName;
-      }
-
+      StringBuilder sb = new StringBuilder();
+      specifiedMetadataAttributes = sDesc.getMetadataAttributes();
       // Sort!
-      java.util.Arrays.sort(sortArray);
+      java.util.Arrays.sort(specifiedMetadataAttributes);
       // Build the metadata string piece now
-      packList(metadataString,sortArray,'+');
+      packList(sb,specifiedMetadataAttributes,'+');
+      metadataString = sb.toString();
     }
     else
       catAccum = new CategoryPathAccumulator(llc);
 
     // Calculate the part of the version string that comes from path name and mapping.
     // This starts with = since ; is used by another optional component (the forced acls)
-    StringBuilder pathNameAttributeVersion = new StringBuilder();
-    if (pathAttributeName != null)
-      pathNameAttributeVersion.append("=").append(pathAttributeName).append(":").append(pathSeparator).append(":").append(matchMap);
-
-    // The version string includes the following:
-    // 1) The modify date for the document
-    // 2) The rights for the document, ordered (which can change without changing the ModifyDate field)
-    // 3) The requested metadata fields (category and attribute, ordered) for the document
-    //
-    // The document identifiers are object id's.
+    String pathNameAttributeVersion;
+    StringBuilder sb2 = new StringBuilder();
+    if (sDesc.getPathAttributeName() != null)
+      sb2.append("=").append(sDesc.getPathAttributeName()).append(":").append(sDesc.getPathSeparator()).append(":").append(sDesc.getMatchMapString());
+    pathNameAttributeVersion = sb2.toString();
 
-    String[] rval = new String[documentIdentifiers.length];
-    i = 0;
-    while (i < documentIdentifiers.length)
+    // Since the identifier indicates it is a directory, then queue up all the current children which pass the filter.
+    String filterString = sDesc.getFilterString();
+
+    for (String documentIdentifier : documentIdentifiers)
     {
       // Since each livelink access is time-consuming, be sure that we abort if the job has gone inactive
       activities.checkJobStillActive();
 
       // Read the document or folder metadata, which includes the ModifyDate
-      String docID = documentIdentifiers[i];
-
+      String docID = documentIdentifier;
+      
       boolean isFolder = docID.startsWith("F");
 
       int colonPos = docID.indexOf(":",1);
@@ -1230,7 +1153,7 @@ public class LivelinkConnector extends o
         vol = new Integer(docID.substring(1,colonPos)).intValue();
       }
 
-      rval[i] = null;
+      getSession();
       ObjectInformation value = llc.getObjectInformation(vol,objID);
       if (value.exists())
       {
@@ -1245,88 +1168,191 @@ public class LivelinkConnector extends o
           {
             // We were able to get rights, so object still exists.
 
-            // I rearranged this on 11/7/2006 so that it would be more parseable, since
-            // we want to pull the transient information out of the version string where
-            // possible (so there is no mismatch between version checking and ingestion).
-
-            StringBuilder sb = new StringBuilder();
-
-            // On 1/17/2008 I changed the version generation code to NOT include metadata, view info, etc. for folders, since
-            // folders make absolutely no use of this info.
-            if (!isFolder)
+            // Changed folder versioning for MCF 2.0
+            if (isFolder)
             {
-              if (includeAllMetadata)
+              // === Livelink folder ===
+              // I'm still not sure if Livelink folder modified dates are one-level or hierarchical.
+              // The code below assumes one-level only, so we always scan folders and there's no versioning
+              if (Logging.connectors.isDebugEnabled())
+                Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
+
+              int sanityRetryCount = FAILURE_RETRY_COUNT;
+              while (true)
+              {
+                ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
+                try
+                {
+                  t.start();
+                  LLValue childrenDocs;
+                  try
+                  {
+                    childrenDocs = t.finishUp();
+                  }
+                  catch (ManifoldCFException e)
+                  {
+                    sanityRetryCount = assessRetry(sanityRetryCount,e);
+                    continue;
+                  }
+
+                  int size = 0;
+                  
+                  if (childrenDocs.isRecord())
+                    size = 1;
+                  if (childrenDocs.isTable())
+                    size = childrenDocs.size();
+
+                  // System.out.println("Total child count = "+Integer.toString(size));
+                  // Do the scan
+                  for (int j = 0; j < size; j++)
+                  {
+                    int childID = childrenDocs.toInteger(j, "ID");
+
+                    if (Logging.connectors.isDebugEnabled())
+                      Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" : ID="+Integer.toString(childID));
+
+                    int subtype = childrenDocs.toInteger(j, "SubType");
+                    boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
+                      subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
+
+                    // If it's a folder, we just let it through for now
+                    if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name") + "." + childrenDocs.toString(j,"FileType"), spec) == false)
+                    {
+                      if (Logging.connectors.isDebugEnabled())
+                        Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" was excluded by inclusion criteria");
+                      continue;
+                    }
+
+                    if (childIsFolder)
+                    {
+                      if (Logging.connectors.isDebugEnabled())
+                        Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a folder, project, or compound document; adding a reference");
+                      if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
+                      {
+                        // If we pick up a project object, we need to describe the volume object (which
+                        // will be the root of all documents beneath)
+                        activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new Integer(-childID).toString());
+                      }
+                      else
+                        activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new Integer(childID).toString());
+                    }
+                    else
+                    {
+                      if (Logging.connectors.isDebugEnabled())
+                        Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a simple document; adding a reference");
+
+                      activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString());
+                    }
+
+                  }
+                  break;
+                }
+                catch (InterruptedException e)
+                {
+                  t.interrupt();
+                  throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+                }
+                catch (RuntimeException e)
+                {
+                  sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true);
+                  continue;
+                }
+              }
+              if (Logging.connectors.isDebugEnabled())
+                Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
+            }
+            else
+            {
+              // === Livelink document ===
+              
+              // The version string includes the following:
+              // 1) The modify date for the document
+              // 2) The rights for the document, ordered (which can change without changing the ModifyDate field)
+              // 3) The requested metadata fields (category and attribute, ordered) for the document
+              //
+              // The document identifiers are object id's.
+      
+              StringBuilder sb = new StringBuilder();
+
+              String[] categoryPaths;
+              if (sDesc.includeAllMetadata())
               {
                 // Find all the metadata associated with this object, and then
                 // find the set of category pathnames that correspond to it.
                 int[] catIDs = getObjectCategoryIDs(vol,objID);
-                String[] categoryPaths = catAccum.getCategoryPathsAttributeNames(catIDs);
+                categoryPaths = catAccum.getCategoryPathsAttributeNames(catIDs);
                 // Sort!
                 java.util.Arrays.sort(categoryPaths);
                 // Build the metadata string piece now
                 packList(sb,categoryPaths,'+');
               }
               else
+              {
+                categoryPaths = specifiedMetadataAttributes;
                 sb.append(metadataString);
-            }
-
-            String[] actualAcls;
-            String denyAcl;
-            if (acls != null && acls.length == 0)
-            {
-              // No forced acls.  Read the actual acls from livelink, as a set of rights.
-              // We need also to add in support for the special rights objects.  These are:
-              // -1: RIGHT_WORLD
-              // -2: RIGHT_SYSTEM
-              // -3: RIGHT_OWNER
-              // -4: RIGHT_GROUP
-              //
-              // RIGHT_WORLD means guest access.
-              // RIGHT_SYSTEM is "Public Access".
-              // RIGHT_OWNER is access by the owner of the object.
-              // RIGHT_GROUP is access by a member of the base group containing the owner
-              //
-              // These objects are returned by the GetObjectRights() call made above, and NOT
-              // returned by LLUser.ListObjects().  We have to figure out how to map these to
-              // things that are
-              // the equivalent of acls.
-
-              actualAcls = lookupTokens(rights, value);
-              java.util.Arrays.sort(actualAcls);
-              // If security is on, no deny acl is needed for the local authority, since the repository does not support "deny".  But this was added
-              // to be really really really sure.
-              denyAcl = denyToken;
+              }
+              
+              String[] actualAcls;
+              String[] denyAcls;
+              
+              String denyAcl;
+              if (acls != null && acls.length == 0)
+              {
+                // No forced acls.  Read the actual acls from livelink, as a set of rights.
+                // We need also to add in support for the special rights objects.  These are:
+                // -1: RIGHT_WORLD
+                // -2: RIGHT_SYSTEM
+                // -3: RIGHT_OWNER
+                // -4: RIGHT_GROUP
+                //
+                // RIGHT_WORLD means guest access.
+                // RIGHT_SYSTEM is "Public Access".
+                // RIGHT_OWNER is access by the owner of the object.
+                // RIGHT_GROUP is access by a member of the base group containing the owner
+                //
+                // These objects are returned by the GetObjectRights() call made above, and NOT
+                // returned by LLUser.ListObjects().  We have to figure out how to map these to
+                // things that are
+                // the equivalent of acls.
+
+                actualAcls = lookupTokens(rights, value);
+                java.util.Arrays.sort(actualAcls);
+                // If security is on, no deny acl is needed for the local authority, since the repository does not support "deny".  But this was added
+                // to be really really really sure.
+                denyAcl = defaultAuthorityDenyToken;
 
-            }
-            else if (acls != null && acls.length > 0)
-            {
-              // Forced acls
-              actualAcls = acls;
-              denyAcl = defaultAuthorityDenyToken;
-            }
-            else
-            {
-              // Security is OFF
-              actualAcls = acls;
-              denyAcl = null;
-            }
+              }
+              else if (acls != null && acls.length > 0)
+              {
+                // Forced acls
+                actualAcls = acls;
+                denyAcl = defaultAuthorityDenyToken;
+              }
+              else
+              {
+                // Security is OFF
+                actualAcls = acls;
+                denyAcl = null;
+              }
 
-            // Now encode the acls.  If null, we write a special value.
-            if (actualAcls == null)
-              sb.append('-');
-            else
-            {
-              sb.append('+');
-              packList(sb,actualAcls,'+');
-              // This was added on 4/21/2008 to support forced acls working with the global default authority.
-              pack(sb,denyAcl,'+');
-            }
+              // Now encode the acls.  If null, we write a special value.
+              if (actualAcls == null)
+              {
+                sb.append('-');
+                denyAcls = null;
+              }
+              else
+              {
+                sb.append('+');
+                packList(sb,actualAcls,'+');
+                // This was added on 4/21/2008 to support forced acls working with the global default authority.
+                pack(sb,denyAcl,'+');
+                denyAcls = new String[]{denyAcl};
+              }
 
-            // The date does not need to be parseable
-            sb.append(dt.toString());
+              // The date does not need to be parseable
+              sb.append(dt.toString());
 
-            if (!isFolder)
-            {
               // PathNameAttributeVersion comes completely from the spec, so we don't
               // have to worry about it changing.  No need, therefore, to parse it during
               // processDocuments.
@@ -1334,33 +1360,60 @@ public class LivelinkConnector extends o
 
               // Tack on ingestCgiPath, to insulate us against changes to the repository connection setup.  Added 9/7/07.
               sb.append("_").append(viewBasePath);
-            }
 
-            rval[i] = sb.toString();
-            if (Logging.connectors.isDebugEnabled())
-              Logging.connectors.debug("Livelink: Successfully calculated version string for object "+Integer.toString(vol)+":"+Integer.toString(objID)+" : '"+rval[i]+"'");
+              String versionString = sb.toString();
+              if (Logging.connectors.isDebugEnabled())
+                Logging.connectors.debug("Livelink: Successfully calculated version string for document "+Integer.toString(vol)+":"+Integer.toString(objID)+" : '"+versionString+"'");
+              
+              if (activities.checkDocumentNeedsReindexing(documentIdentifier,versionString))
+              {
+                // Index the document
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
+                if (checkIngest(llc,objID,spec))
+                {
+                  if (Logging.connectors.isDebugEnabled())
+                    Logging.connectors.debug("Livelink: Decided to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID));
+
+                  // Grab the access tokens for this file from the version string, inside ingest method.
+                  ingestFromLiveLink(llc,documentIdentifier,versionString,actualAcls,denyAcls,categoryPaths,activities,desc,sDesc);
+                }
+                else
+                {
+                  if (Logging.connectors.isDebugEnabled())
+                    Logging.connectors.debug("Livelink: Decided not to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID)+" - Did not match ingestion criteria");
+                  activities.noDocument(documentIdentifier,versionString);
+                }
+                if (Logging.connectors.isDebugEnabled())
+                  Logging.connectors.debug("Livelink: Done processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
+              }
+            }
           }
           else
           {
             if (Logging.connectors.isDebugEnabled())
               Logging.connectors.debug("Livelink: Could not get rights for object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
+            activities.deleteDocument(documentIdentifier);
+            continue;
           }
         }
         else
         {
           if (Logging.connectors.isDebugEnabled())
             Logging.connectors.debug("Livelink: Crawl user cannot see contents of object "+Integer.toString(vol)+":"+Integer.toString(objID)+" - deleting");
+          activities.deleteDocument(documentIdentifier);
+          continue;
         }
       }
       else
       {
         if (Logging.connectors.isDebugEnabled())
           Logging.connectors.debug("Livelink: Object "+Integer.toString(vol)+":"+Integer.toString(objID)+" has no information - deleting");
+        activities.deleteDocument(documentIdentifier);
+        continue;
       }
-      i++;
 
     }
-    return rval;
   }
 
   protected class ListObjectsThread extends Thread
@@ -1418,186 +1471,6 @@ public class LivelinkConnector extends o
     }
   }
 
-  /** Process a set of documents.
-  * This is the method that should cause each document to be fetched, processed, and the results either added
-  * to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
-  * The document specification allows this class to filter what is done based on the job.
-  *@param documentIdentifiers is the set of document identifiers to process.
-  *@param activities is the interface this method should use to queue up new document references
-  * and ingest documents.
-  *@param spec is the document specification.
-  *@param scanOnly is an array corresponding to the document identifiers.  It is set to true to indicate when the processing
-  * should only find other references, and should not actually call the ingestion methods.
-  */
-  @Override
-  public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities, DocumentSpecification spec, boolean[] scanOnly)
-    throws ManifoldCFException, ServiceInterruption
-  {
-    getSession();
-
-    // Initialize livelink context
-    LivelinkContext llc = new LivelinkContext();
-    
-    // First, initialize the table of catid's.
-    // Keeping this around will allow us to benefit from batching of documents.
-    MetadataDescription desc = new MetadataDescription(llc);
-
-    // Build the node/path cache
-    SystemMetadataDescription sDesc = new SystemMetadataDescription(llc,spec);
-
-    int i = 0;
-    while (i < documentIdentifiers.length)
-    {
-      // Since each livelink access is time-consuming, be sure that we abort if the job has gone inactive
-      activities.checkJobStillActive();
-      
-      String documentIdentifier = documentIdentifiers[i];
-      String version = versions[i];
-      
-      boolean doScanOnly = scanOnly[i];
-
-      boolean isFolder = documentIdentifier.startsWith("F");
-      int colonPosition = documentIdentifier.indexOf(":",1);
-      int vol;
-      int objID;
-      if (colonPosition == -1)
-      {
-        vol = LLENTWK_VOL;
-        objID = new Integer(documentIdentifier.substring(1)).intValue();
-      }
-      else
-      {
-        vol = new Integer(documentIdentifier.substring(1,colonPosition)).intValue();
-        objID = new Integer(documentIdentifier.substring(colonPosition+1)).intValue();
-      }
-
-      if (isFolder)
-      {
-        // We don't know if LL will let us version folders properly, so ALWAYS process folders.
-        if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("Livelink: Processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
-
-        // Since the identifier indicates it is a directory, then queue up all the current children which pass the filter.
-        String filterString = buildFilterString(spec);
-
-        int sanityRetryCount = FAILURE_RETRY_COUNT;
-        while (true)
-        {
-          ListObjectsThread t = new ListObjectsThread(vol,objID,filterString);
-          try
-          {
-            t.start();
-            LLValue childrenDocs;
-            try
-            {
-              childrenDocs = t.finishUp();
-            }
-            catch (ManifoldCFException e)
-            {
-              sanityRetryCount = assessRetry(sanityRetryCount,e);
-              continue;
-            }
-
-            int size = 0;
-            
-            if (childrenDocs.isRecord())
-              size = 1;
-            if (childrenDocs.isTable())
-              size = childrenDocs.size();
-
-            // System.out.println("Total child count = "+Integer.toString(size));
-            // Do the scan
-            int j = 0;
-            while (j < size)
-            {
-              int childID = childrenDocs.toInteger(j, "ID");
-
-              if (Logging.connectors.isDebugEnabled())
-                Logging.connectors.debug("Livelink: Found a child of folder "+Integer.toString(vol)+":"+Integer.toString(objID)+" : ID="+Integer.toString(childID));
-
-              int subtype = childrenDocs.toInteger(j, "SubType");
-              boolean childIsFolder = (subtype == LAPI_DOCUMENTS.FOLDERSUBTYPE || subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE ||
-                subtype == LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE);
-
-              // If it's a folder, we just let it through for now
-              if (!childIsFolder && checkInclude(childrenDocs.toString(j,"Name") + "." + childrenDocs.toString(j,"FileType"), spec) == false)
-              {
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" was excluded by inclusion criteria");
-                j++;
-                continue;
-              }
-
-              if (childIsFolder)
-              {
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a folder, project, or compound document; adding a reference");
-                if (subtype == LAPI_DOCUMENTS.PROJECTSUBTYPE)
-                {
-                  // If we pick up a project object, we need to describe the volume object (which
-                  // will be the root of all documents beneath)
-                  activities.addDocumentReference("F"+new Integer(childID).toString()+":"+new Integer(-childID).toString());
-                }
-                else
-                  activities.addDocumentReference("F"+new Integer(vol).toString()+":"+new Integer(childID).toString());
-              }
-              else
-              {
-                if (Logging.connectors.isDebugEnabled())
-                  Logging.connectors.debug("Livelink: Child identifier "+Integer.toString(childID)+" is a simple document; adding a reference");
-
-                activities.addDocumentReference("D"+new Integer(vol).toString()+":"+new Integer(childID).toString());
-              }
-
-              j++;
-            }
-            break;
-          }
-          catch (InterruptedException e)
-          {
-            t.interrupt();
-            throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-          }
-          catch (RuntimeException e)
-          {
-            sanityRetryCount = handleLivelinkRuntimeException(e,sanityRetryCount,true);
-            continue;
-          }
-        }
-        if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("Livelink: Done processing folder "+Integer.toString(vol)+":"+Integer.toString(objID));
-      }
-      else
-      {
-        // It's a known file, and we've already checked whether it's allowed or not (except any
-        // checks based on the file data)
-
-        if (doScanOnly == false)
-        {
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
-          if (checkIngest(llc,objID,spec))
-          {
-            if (Logging.connectors.isDebugEnabled())
-              Logging.connectors.debug("Livelink: Decided to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID));
-
-            // Grab the access tokens for this file from the version string, inside ingest method.
-            ingestFromLiveLink(llc,documentIdentifiers[i],version,activities,desc,sDesc);
-          }
-          else
-          {
-            if (Logging.connectors.isDebugEnabled())
-              Logging.connectors.debug("Livelink: Decided not to ingest document "+Integer.toString(vol)+":"+Integer.toString(objID)+" - Did not match ingestion criteria");
-
-          }
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("Livelink: Done processing document "+Integer.toString(vol)+":"+Integer.toString(objID));
-        }
-      }
-      i++;
-    }
-  }
-
   /** Get the maximum number of documents to amalgamate together into one batch, for this connector.
   *@return the maximum number. 0 indicates "unlimited".
   */
@@ -4269,6 +4142,8 @@ public class LivelinkConnector extends o
   */
   protected void ingestFromLiveLink(LivelinkContext llc,
     String documentIdentifier, String version,
+    String[] actualAcls, String[] denyAcls,
+    String[] categoryPaths,
     IProcessActivity activities,
     MetadataDescription desc, SystemMetadataDescription sDesc)
     throws ManifoldCFException, ServiceInterruption
@@ -4373,16 +4248,10 @@ public class LivelinkConnector extends o
             // Iterate over the metadata items.  These are organized by category
             // for speed of lookup.
 
-            // Unpack version string
-            int startPos = 0;
-
-            // Metadata items first
-            ArrayList metadataItems = new ArrayList();
-            startPos = unpackList(metadataItems,version,startPos,'+');
-            Iterator catIter = desc.getItems(metadataItems);
+            Iterator<MetadataItem> catIter = desc.getItems(categoryPaths);
             while (catIter.hasNext())
             {
-              MetadataItem item = (MetadataItem)catIter.next();
+              MetadataItem item = catIter.next();
               MetadataPathItem pathItem = item.getPathItem();
               if (pathItem != null)
               {
@@ -4392,10 +4261,10 @@ public class LivelinkConnector extends o
                 if (catVersion != null)
                 {
                   // Go through attributes now
-                  Iterator attrIter = item.getAttributeNames();
+                  Iterator<String> attrIter = item.getAttributeNames();
                   while (attrIter.hasNext())
                   {
-                    String attrName = (String)attrIter.next();
+                    String attrName = attrIter.next();
                     // Create a unique metadata name
                     String metadataName = pathItem.getCatName()+":"+attrName;
                     // Fetch the metadata and stuff it into the RepositoryData structure
@@ -4410,32 +4279,8 @@ public class LivelinkConnector extends o
               }
             }
 
-            // Unpack acls (conditionally)
-            if (startPos < version.length())
-            {
-              char x = version.charAt(startPos++);
-              if (x == '+')
-              {
-                ArrayList acls = new ArrayList();
-                startPos = unpackList(acls,version,startPos,'+');
-                // Turn into acls and add into description
-                String[] aclArray = new String[acls.size()];
-                int j = 0;
-                while (j < aclArray.length)
-                {
-                  aclArray[j] = (String)acls.get(j);
-                  j++;
-                }
-
-                StringBuilder denyBuffer = new StringBuilder();
-                startPos = unpack(denyBuffer,version,startPos,'+');
-                String denyAcl = denyBuffer.toString();
-                String[] denyAclArray = new String[1];
-                denyAclArray[0] = denyAcl;
-                
-                rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,aclArray,denyAclArray);
-              }
-            }
+            if (actualAcls != null && denyAcls != null)
+              rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT,actualAcls,denyAcls);
 
             // Add the path metadata item into the mix, if enabled
             String pathAttributeName = sDesc.getPathAttributeName();
@@ -5516,11 +5361,11 @@ public class LivelinkConnector extends o
   protected class LivelinkContext
   {
     /** Cache of ObjectInformation objects. */
-    protected Map<ObjectInformation,ObjectInformation> objectInfoMap = new HashMap<ObjectInformation,ObjectInformation>();
+    protected final Map<ObjectInformation,ObjectInformation> objectInfoMap = new HashMap<ObjectInformation,ObjectInformation>();
     /** Cache of VersionInformation objects. */
-    protected Map<VersionInformation,VersionInformation> versionInfoMap = new HashMap<VersionInformation,VersionInformation>();
+    protected final Map<VersionInformation,VersionInformation> versionInfoMap = new HashMap<VersionInformation,VersionInformation>();
     /** Cache of UserInformation objects */
-    protected Map<UserInformation,UserInformation> userInfoMap = new HashMap<UserInformation,UserInformation>();
+    protected final Map<UserInformation,UserInformation> userInfoMap = new HashMap<UserInformation,UserInformation>();
     
     public LivelinkContext()
     {
@@ -6706,101 +6551,12 @@ public class LivelinkConnector extends o
 
   // Protected static methods
 
-  /** Convert the document specification to a set of query clauses meant to match file names.
-  *@param spec is the specification string.
-  *@return the correct part of the filter string.  Empty string is returned if there is no filtering.
-  */
-  protected static String buildFilterString(DocumentSpecification spec)
-  {
-    StringBuilder rval = new StringBuilder();
-
-
-    boolean first = true;
-    int i = 0;
-    while (i < spec.getChildCount())
-    {
-      SpecificationNode sn = spec.getChild(i++);
-      if (sn.getType().equals("include"))
-      {
-        String includeMatch = sn.getAttributeValue("filespec");
-        if (includeMatch != null)
-        {
-          // Peel off the extension
-          int index = includeMatch.lastIndexOf(".");
-          if (index != -1)
-          {
-            String type = includeMatch.substring(index+1).toLowerCase().replace('*','%');
-            if (first)
-              first = false;
-            else
-              rval.append(" or ");
-            rval.append("lower(FileType) like '").append(type).append("'");
-          }
-        }
-      }
-    }
-    String filterStringPiece = rval.toString();
-    if (filterStringPiece.length() == 0)
-      return "0>1";
-    StringBuilder sb = new StringBuilder();
-    sb.append("SubType=").append(new Integer(LAPI_DOCUMENTS.FOLDERSUBTYPE).toString());
-    sb.append(" or SubType=").append(new Integer(LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE).toString());
-    sb.append(" or SubType=").append(new Integer(LAPI_DOCUMENTS.PROJECTSUBTYPE).toString());
-    sb.append(" or (SubType=").append(new Integer(LAPI_DOCUMENTS.DOCUMENTSUBTYPE).toString());
-    sb.append(" and (");
-    // Walk through the document spec to find the documents that match under the specified root
-    // include lower(column)=spec
-    sb.append(filterStringPiece);
-    sb.append("))");
-    return sb.toString();
-  }
-
-
-  /** Grab forced acl out of document specification.
-  *@param spec is the document specification.
-  *@return the acls.
-  */
-  protected static String[] getAcls(DocumentSpecification spec)
-  {
-    HashMap map = new HashMap();
-    int i = 0;
-    boolean securityOn = true;
-    while (i < spec.getChildCount())
-    {
-      SpecificationNode sn = spec.getChild(i++);
-      if (sn.getType().equals("access"))
-      {
-        String token = sn.getAttributeValue("token");
-        map.put(token,token);
-      }
-      else if (sn.getType().equals("security"))
-      {
-        String value = sn.getAttributeValue("value");
-        if (value.equals("on"))
-          securityOn = true;
-        else if (value.equals("off"))
-          securityOn = false;
-      }
-    }
-    if (!securityOn)
-      return null;
-
-    String[] rval = new String[map.size()];
-    Iterator iter = map.keySet().iterator();
-    i = 0;
-    while (iter.hasNext())
-    {
-      rval[i++] = (String)iter.next();
-    }
-    return rval;
-  }
-
   /** Check if a file or directory should be included, given a document specification.
   *@param filename is the name of the "file".
   *@param documentSpecification is the specification.
   *@return true if it should be included.
   */
-  protected static boolean checkInclude(String filename, DocumentSpecification documentSpecification)
+  protected static boolean checkInclude(String filename, Specification documentSpecification)
     throws ManifoldCFException
   {
     // Scan includes to insure we match
@@ -6844,7 +6600,7 @@ public class LivelinkConnector extends o
   *@param objID is the file ID.
   *@param documentSpecification is the specification.
   */
-  protected boolean checkIngest(LivelinkContext llc, int objID, DocumentSpecification documentSpecification)
+  protected boolean checkIngest(LivelinkContext llc, int objID, Specification documentSpecification)
     throws ManifoldCFException
   {
     // Since the only exclusions at this point are not based on file contents, this is a no-op.
@@ -6932,8 +6688,8 @@ public class LivelinkConnector extends o
   */
   protected static class VolumeAndId
   {
-    protected int volumeID;
-    protected int folderID;
+    protected final int volumeID;
+    protected final int folderID;
 
     public VolumeAndId(int volumeID, int folderID)
     {
@@ -6956,8 +6712,8 @@ public class LivelinkConnector extends o
   */
   protected static class MetadataPathItem
   {
-    protected int catID;
-    protected String catName;
+    protected final int catID;
+    protected final String catName;
 
     /** Constructor.
     */
@@ -6989,8 +6745,8 @@ public class LivelinkConnector extends o
   */
   protected static class MetadataItem
   {
-    protected MetadataPathItem pathItem;
-    protected Map attributeNames = new HashMap();
+    protected final MetadataPathItem pathItem;
+    protected final Set<String> attributeNames = new HashSet<String>();
 
     /** Constructor.
     */
@@ -7003,7 +6759,7 @@ public class LivelinkConnector extends o
     */
     public void addAttribute(String attributeName)
     {
-      attributeNames.put(attributeName,attributeName);
+      attributeNames.add(attributeName);
     }
 
     /** Get the path object.
@@ -7017,9 +6773,9 @@ public class LivelinkConnector extends o
     /** Get an iterator over the attribute names.
     *@return the iterator.
     */
-    public Iterator getAttributeNames()
+    public Iterator<String> getAttributeNames()
     {
-      return attributeNames.keySet().iterator();
+      return attributeNames.iterator();
     }
 
   }
@@ -7033,28 +6789,42 @@ public class LivelinkConnector extends o
     protected final LivelinkContext llc;
     
     // The path attribute name
-    protected String pathAttributeName;
+    protected final String pathAttributeName;
 
     // The path separator
-    protected String pathSeparator;
+    protected final String pathSeparator;
     
     // The node ID to path name mapping (which acts like a cache)
-    protected Map pathMap = new HashMap();
+    protected final Map<String,String> pathMap = new HashMap<String,String>();
 
     // The path name map
-    protected MatchMap matchMap = new MatchMap();
+    protected final MatchMap matchMap = new MatchMap();
+
+    // Acls
+    protected final Set<String> aclMap = new HashSet<String>();
+    protected final boolean securityOn;
+    
+    // Filter string
+    protected final String filterString;
+    
+    protected final Set<String> holder = new HashSet<String>();
+    protected final boolean includeAllMetadata;
 
     /** Constructor */
-    public SystemMetadataDescription(LivelinkContext llc, DocumentSpecification spec)
-      throws ManifoldCFException
+    public SystemMetadataDescription(LivelinkContext llc, Specification spec)
+      throws ManifoldCFException, ServiceInterruption
     {
       this.llc = llc;
-      pathAttributeName = null;
-      pathSeparator = null;
-      int i = 0;
-      while (i < spec.getChildCount())
+      String pathAttributeName = null;
+      String pathSeparator = null;
+      boolean securityOn = true;
+      StringBuilder fsb = new StringBuilder();
+      boolean first = true;
+      boolean includeAllMetadata = false;
+      
+      for (int i = 0; i < spec.getChildCount(); i++)
       {
-        SpecificationNode n = spec.getChild(i++);
+        SpecificationNode n = spec.getChild(i);
         if (n.getType().equals("pathnameattribute"))
         {
           pathAttributeName = n.getAttributeValue("value");
@@ -7068,9 +6838,133 @@ public class LivelinkConnector extends o
           String pathReplace = n.getAttributeValue("replace");
           matchMap.appendMatchPair(pathMatch,pathReplace);
         }
+        else if (n.getType().equals("access"))
+        {
+          String token = n.getAttributeValue("token");
+          aclMap.add(token);
+        }
+        else if (n.getType().equals("security"))
+        {
+          String value = n.getAttributeValue("value");
+          if (value.equals("on"))
+            securityOn = true;
+          else if (value.equals("off"))
+            securityOn = false;
+        }
+        else if (n.getType().equals("include"))
+        {
+          String includeMatch = n.getAttributeValue("filespec");
+          if (includeMatch != null)
+          {
+            // Peel off the extension
+            int index = includeMatch.lastIndexOf(".");
+            if (index != -1)
+            {
+              String type = includeMatch.substring(index+1).toLowerCase().replace('*','%');
+              if (first)
+                first = false;
+              else
+                fsb.append(" or ");
+              fsb.append("lower(FileType) like '").append(type).append("'");
+            }
+          }
+        }
+        else if (n.getType().equals("allmetadata"))
+        {
+          String isAll = n.getAttributeValue("all");
+          if (isAll != null && isAll.equals("true"))
+            includeAllMetadata = true;
+        }
+        else if (n.getType().equals("metadata"))
+        {
+          String category = n.getAttributeValue("category");
+          String attributeName = n.getAttributeValue("attribute");
+          String isAll = n.getAttributeValue("all");
+          if (isAll != null && isAll.equals("true"))
+          {
+            // Locate all metadata items for the specified category path,
+            // and enter them into the array
+            getSession();
+            String[] attrs = getCategoryAttributes(llc,category);
+            if (attrs != null)
+            {
+              int j = 0;
+              while (j < attrs.length)
+              {
+                attributeName = attrs[j++];
+                String metadataName = packCategoryAttribute(category,attributeName);
+                holder.add(metadataName);
+              }
+            }
+          }
+          else
+          {
+            String metadataName = packCategoryAttribute(category,attributeName);
+            holder.add(metadataName);
+          }
+
+        }
+      }
+      
+      this.includeAllMetadata = includeAllMetadata;
+      this.pathAttributeName = pathAttributeName;
+      this.pathSeparator = pathSeparator;
+      this.securityOn = securityOn;
+      String filterStringPiece = fsb.toString();
+      if (filterStringPiece.length() == 0)
+        this.filterString = "0>1";
+      else
+      {
+        StringBuilder sb = new StringBuilder();
+        sb.append("SubType=").append(new Integer(LAPI_DOCUMENTS.FOLDERSUBTYPE).toString());
+        sb.append(" or SubType=").append(new Integer(LAPI_DOCUMENTS.COMPOUNDDOCUMENTSUBTYPE).toString());
+        sb.append(" or SubType=").append(new Integer(LAPI_DOCUMENTS.PROJECTSUBTYPE).toString());
+        sb.append(" or (SubType=").append(new Integer(LAPI_DOCUMENTS.DOCUMENTSUBTYPE).toString());
+        sb.append(" and (");
+        // Walk through the document spec to find the documents that match under the specified root
+        // include lower(column)=spec
+        sb.append(filterStringPiece);
+        sb.append("))");
+        this.filterString = sb.toString();
       }
     }
 
+    public boolean includeAllMetadata()
+    {
+      return includeAllMetadata;
+    }
+    
+    public String[] getMetadataAttributes()
+    {
+      // Put into an array
+      String[] specifiedMetadataAttributes = new String[holder.size()];
+      int i = 0;
+      for (String attrName : holder)
+      {
+        specifiedMetadataAttributes[i++] = attrName;
+      }
+      return specifiedMetadataAttributes;
+    }
+    
+    public String getFilterString()
+    {
+      return filterString;
+    }
+    
+    public String[] getAcls()
+    {
+      if (!securityOn)
+        return null;
+
+      String[] rval = new String[aclMap.size()];
+      int i = 0;
+      for (String token : aclMap)
+      {
+        rval[i++] = token;
+      }
+      return rval;
+    }
+  
     /** Get the path attribute name.
     *@return the path attribute name, or null if none specified.
     */
@@ -7079,6 +6973,13 @@ public class LivelinkConnector extends o
       return pathAttributeName;
     }
 
+    /** Get the path separator.
+    */
+    public String getPathSeparator()
+    {
+      return pathSeparator;
+    }
+    
     /** Given an identifier, get the translated string that goes into the metadata.
     */
     public String getPathAttributeValue(String documentIdentifier)
@@ -7090,6 +6991,13 @@ public class LivelinkConnector extends o
       return matchMap.translate(path);
     }
 
+    /** Get the matchmap string.
+    */
+    public String getMatchMapString()
+    {
+      return matchMap.toString();
+    }
+    
     /** For a given node, get its path.
     */
     public String getNodePathString(String documentIdentifier)
@@ -7097,7 +7005,7 @@ public class LivelinkConnector extends o
     {
       if (Logging.connectors.isDebugEnabled())
         Logging.connectors.debug("Looking up path for '"+documentIdentifier+"'");
-      String path = (String)pathMap.get(documentIdentifier);
+      String path = pathMap.get(documentIdentifier);
       if (path == null)
       {
         // Not yet present.  Look it up, recursively
@@ -7174,7 +7082,7 @@ public class LivelinkConnector extends o
     protected final LivelinkContext llc;
     
     // This is a map of category name to category ID and attributes
-    protected Map categoryMap = new HashMap();
+    protected final Map<String,MetadataPathItem> categoryMap = new HashMap<String,MetadataPathItem>();
 
     /** Constructor.
     */
@@ -7186,21 +7094,19 @@ public class LivelinkConnector extends o
     /** Iterate over the metadata items represented by the specified chunk of version string.
     *@return an iterator over MetadataItem objects.
     */
-    public Iterator getItems(ArrayList metadataItems)
+    public Iterator<MetadataItem> getItems(String[] metadataItems)
       throws ManifoldCFException, ServiceInterruption
     {
       // This is the map that will be iterated over for a return value.
       // It gets built out of (hopefully cached) data from categoryMap.
-      HashMap newMap = new HashMap();
+      Map<String,MetadataItem> newMap = new HashMap<String,MetadataItem>();
 
       // Start at root
       ObjectInformation rootValue = null;
 
       // Walk through string and process each metadata element in turn.
-      int i = 0;
-      while (i < metadataItems.size())
+      for (String metadataSpec : metadataItems)
       {
-        String metadataSpec = (String)metadataItems.get(i++);
         StringBuilder categoryBuffer = new StringBuilder();
         StringBuilder attributeBuffer = new StringBuilder();
         unpackCategoryAttribute(categoryBuffer,attributeBuffer,metadataSpec);
@@ -7208,12 +7114,12 @@ public class LivelinkConnector extends o
         String attributeName = attributeBuffer.toString();
 
         // If there's already an entry for this category in the return map, use it
-        MetadataItem mi = (MetadataItem)newMap.get(category);
+        MetadataItem mi = newMap.get(category);
         if (mi == null)
         {
           // Now, look up the node information
           // Convert category to cat id.
-          MetadataPathItem item = (MetadataPathItem)categoryMap.get(category);
+          MetadataPathItem item = categoryMap.get(category);
           if (item == null)
           {
             RootValue rv = new RootValue(llc,category);
@@ -7256,11 +7162,11 @@ public class LivelinkConnector extends o
     
     // This is the map from category ID to category path name.
     // It's keyed by an Integer formed from the id, and has String values.
-    protected HashMap categoryPathMap = new HashMap();
+    protected final Map<Integer,String> categoryPathMap = new HashMap<Integer,String>();
 
     // This is the map from category ID to attribute names.  Keyed
     // by an Integer formed from the id, and has a String[] value.
-    protected HashMap attributeMap = new HashMap();
+    protected final Map<Integer,String[]> attributeMap = new HashMap<Integer,String[]>();
 
     /** Constructor */
     public CategoryPathAccumulator(LivelinkContext llc)
@@ -7272,12 +7178,11 @@ public class LivelinkConnector extends o
     public String[] getCategoryPathsAttributeNames(int[] catIDs)
       throws ManifoldCFException, ServiceInterruption
     {
-      HashMap set = new HashMap();
-      int i = 0;
-      while (i < catIDs.length)
+      Set<String> set = new HashSet<String>();
+      for (int x : catIDs)
       {
-        Integer key = new Integer(catIDs[i++]);
-        String pathValue = (String)categoryPathMap.get(key);
+        Integer key = new Integer(x);
+        String pathValue = categoryPathMap.get(key);
         if (pathValue == null)
         {
           // Chase the path back up the chain
@@ -7286,7 +7191,7 @@ public class LivelinkConnector extends o
             continue;
           categoryPathMap.put(key,pathValue);
         }
-        String[] attributeNames = (String[])attributeMap.get(key);
+        String[] attributeNames = attributeMap.get(key);
         if (attributeNames == null)
         {
           // Get the attributes for this category
@@ -7296,20 +7201,18 @@ public class LivelinkConnector extends o
           attributeMap.put(key,attributeNames);
         }
         // Now, put the path and the attributes into the hash.
-        int j = 0;
-        while (j < attributeNames.length)
+        for (String attributeName : attributeNames)
         {
-          String metadataName = packCategoryAttribute(pathValue,attributeNames[j++]);
-          set.put(metadataName,metadataName);
+          String metadataName = packCategoryAttribute(pathValue,attributeName);
+          set.add(metadataName);
         }
       }
 
       String[] rval = new String[set.size()];
-      i = 0;
-      Iterator iter = set.keySet().iterator();
-      while (iter.hasNext())
+      int i = 0;
+      for (String value : set)
       {
-        rval[i++] = (String)iter.next();
+        rval[i++] = value;
       }
 
       return rval;
@@ -7380,9 +7283,9 @@ public class LivelinkConnector extends o
   protected class RootValue
   {
     protected final LivelinkContext llc;
-    protected String workspaceName;
+    protected final String workspaceName;
     protected ObjectInformation rootValue = null;
-    protected String remainderPath;
+    protected final String remainderPath;
 
     /** Constructor.
     *@param pathString is the path string.



Mime
View raw message