manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1642163 - in /manifoldcf/trunk/framework: agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/ agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/ pull-agent/src/main/java/org/apache/manifoldcf/author...
Date Thu, 27 Nov 2014 14:48:40 GMT
Author: kwright
Date: Thu Nov 27 14:48:39 2014
New Revision: 1642163

URL: http://svn.apache.org/r1642163
Log:
Refactor CONNECTORS-1116 fix

Modified:
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1642163&r1=1642162&r2=1642163&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
Thu Nov 27 14:48:39 2014
@@ -241,31 +241,27 @@ public class OutputConnectionManager ext
     IOutputConnection[] rval = new IOutputConnection[names.length];
     if (names.length == 0)
       return rval;
-    List<String> fetchNames = new ArrayList<String>();
+    int inputIndex = 0;
     int outputIndex = 0;
-    for (String name : names)
+    while (names.length - inputIndex > FETCH_MAX)
     {
-      if (fetchNames.size() == FETCH_MAX)
-      {
-        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
-        fetchNames.clear();
-      }
-      fetchNames.add(name);
+      outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+      inputIndex += FETCH_MAX;
     }
-    loadMultipleInternal(rval,outputIndex,fetchNames);
+    loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
     return rval;
   }
   
-  protected int loadMultipleInternal(IOutputConnection[] rval, int outputIndex, List<String>
fetchNames)
+  protected int loadMultipleInternal(IOutputConnection[] rval, int outputIndex, String[]
fetchNames, int inputIndex, int length)
     throws ManifoldCFException
   {
     // Build description objects
-    OutputConnectionDescription[] objectDescriptions = new OutputConnectionDescription[fetchNames.size()];
+    OutputConnectionDescription[] objectDescriptions = new OutputConnectionDescription[length];
     StringSetBuffer ssb = new StringSetBuffer();
-    for (int i = 0; i < fetchNames.size(); i++)
+    for (int i = 0; i < length; i++)
     {
       ssb.clear();
-      String name = fetchNames.get(i);
+      String name = fetchNames[inputIndex + i];
       ssb.add(getOutputConnectionKey(name));
       objectDescriptions[i] = new OutputConnectionDescription(name,new StringSet(ssb));
     }

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java?rev=1642163&r1=1642162&r2=1642163&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/transformationconnection/TransformationConnectionManager.java
Thu Nov 27 14:48:39 2014
@@ -241,32 +241,27 @@ public class TransformationConnectionMan
     ITransformationConnection[] rval = new ITransformationConnection[names.length];
     if (names.length == 0)
       return rval;
-    List<String> fetchNames = new ArrayList<String>();
+    int inputIndex = 0;
     int outputIndex = 0;
-    for (String name : names)
+    while (names.length - inputIndex > FETCH_MAX)
     {
-      if (fetchNames.size() == FETCH_MAX)
-      {
-        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
-        fetchNames.clear();
-      }
-      fetchNames.add(name);
+      outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+      inputIndex += FETCH_MAX;
     }
-    loadMultipleInternal(rval,outputIndex,fetchNames);
+    loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
     return rval;
   }
   
-  protected int loadMultipleInternal(ITransformationConnection[] rval, int outputIndex, List<String>
fetchNames)
+  protected int loadMultipleInternal(ITransformationConnection[] rval, int outputIndex, String[]
fetchNames, int inputIndex, int length)
     throws ManifoldCFException
   {
-
     // Build description objects
-    TransformationConnectionDescription[] objectDescriptions = new TransformationConnectionDescription[fetchNames.size()];
+    TransformationConnectionDescription[] objectDescriptions = new TransformationConnectionDescription[length];
     StringSetBuffer ssb = new StringSetBuffer();
-    for (int i = 0; i < fetchNames.size(); i++)
+    for (int i = 0; i < length; i++)
     {
       ssb.clear();
-      String name = fetchNames.get(i);
+      String name = fetchNames[inputIndex + i];
       ssb.add(getTransformationConnectionKey(name));
       objectDescriptions[i] = new TransformationConnectionDescription(name,new StringSet(ssb));
     }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1642163&r1=1642162&r2=1642163&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
Thu Nov 27 14:48:39 2014
@@ -336,31 +336,27 @@ public class AuthorityConnectionManager 
     IAuthorityConnection[] rval = new IAuthorityConnection[names.length];
     if (names.length == 0)
       return rval;
-    List<String> fetchNames = new ArrayList<String>();
+    int inputIndex = 0;
     int outputIndex = 0;
-    for (String name : names)
+    while (names.length - inputIndex > FETCH_MAX)
     {
-      if (fetchNames.size() == FETCH_MAX)
-      {
-        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
-        fetchNames.clear();
-      }
-      fetchNames.add(name);
+      outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+      inputIndex += FETCH_MAX;
     }
-    loadMultipleInternal(rval,outputIndex,fetchNames);
+    loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
     return rval;
   }
   
-  protected int loadMultipleInternal(IAuthorityConnection[] rval, int outputIndex, List<String>
fetchNames)
+  protected int loadMultipleInternal(IAuthorityConnection[] rval, int outputIndex, String[]
fetchNames, int inputIndex, int length)
     throws ManifoldCFException
   {
     // Build description objects
-    AuthorityConnectionDescription[] objectDescriptions = new AuthorityConnectionDescription[fetchNames.size()];
+    AuthorityConnectionDescription[] objectDescriptions = new AuthorityConnectionDescription[length];
     StringSetBuffer ssb = new StringSetBuffer();
-    for (int i = 0; i < fetchNames.size(); i++)
+    for (int i = 0; i < length; i++)
     {
       ssb.clear();
-      String name = fetchNames.get(i);
+      String name = fetchNames[inputIndex + i];
       ssb.add(getAuthorityConnectionKey(name));
       objectDescriptions[i] = new AuthorityConnectionDescription(name,new StringSet(ssb));
     }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java?rev=1642163&r1=1642162&r2=1642163&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/mapping/MappingConnectionManager.java
Thu Nov 27 14:48:39 2014
@@ -345,31 +345,27 @@ public class MappingConnectionManager ex
     IMappingConnection[] rval = new IMappingConnection[names.length];
     if (names.length == 0)
       return rval;
-    List<String> fetchNames = new ArrayList<String>();
+    int inputIndex = 0;
     int outputIndex = 0;
-    for (String name : names)
+    while (names.length - inputIndex > FETCH_MAX)
     {
-      if (fetchNames.size() == FETCH_MAX)
-      {
-        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
-        fetchNames.clear();
-      }
-      fetchNames.add(name);
+      outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+      inputIndex += FETCH_MAX;
     }
-    loadMultipleInternal(rval,outputIndex,fetchNames);
+    loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
     return rval;
   }
   
-  protected int loadMultipleInternal(IMappingConnection[] rval, int outputIndex, List<String>
fetchNames)
+  protected int loadMultipleInternal(IMappingConnection[] rval, int outputIndex, String[]
fetchNames, int inputIndex, int length)
     throws ManifoldCFException
   {
     // Build description objects
-    MappingConnectionDescription[] objectDescriptions = new MappingConnectionDescription[fetchNames.size()];
+    MappingConnectionDescription[] objectDescriptions = new MappingConnectionDescription[length];
     StringSetBuffer ssb = new StringSetBuffer();
-    for (int i = 0; i < fetchNames.size(); i++)
+    for (int i = 0; i < length; i++)
     {
       ssb.clear();
-      String name = fetchNames.get(i);
+      String name = fetchNames[inputIndex + i];
       ssb.add(getMappingConnectionKey(name));
       objectDescriptions[i] = new MappingConnectionDescription(name,new StringSet(ssb));
     }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1642163&r1=1642162&r2=1642163&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
Thu Nov 27 14:48:39 2014
@@ -648,7 +648,7 @@ public class Jobs extends org.apache.man
   }
 
   // Only fetch 200 jobs at a time, for resource reasons
-  protected final int MAX_FETCH = 200;
+  protected final int FETCH_MAX = 200;
 
   /** Get a list of all jobs which are not in the process of being deleted already.
   *@return the array of all jobs.
@@ -825,34 +825,27 @@ public class Jobs extends org.apache.man
     IJobDescription[] rval = new IJobDescription[ids.length];
     if (ids.length == 0)
       return rval;
-    
-    List<Long> idsToDo = new ArrayList<Long>();
-    List<Boolean> readOnliesToDo = new ArrayList<Boolean>();
+
+    int inputIndex = 0;
     int outputIndex = 0;
-    for (int i = 0; i < ids.length; i++)
+    while (ids.length - inputIndex > FETCH_MAX)
     {
-      if (idsToDo.size() == MAX_FETCH)
-      {
-        outputIndex = loadMultipleInternal(rval,outputIndex,idsToDo,readOnliesToDo);
-        idsToDo.clear();
-        readOnliesToDo.clear();
-      }
-      idsToDo.add(ids[i]);
-      readOnliesToDo.add(readOnlies[i]);
+      outputIndex = loadMultipleInternal(rval,outputIndex,ids,readOnlies,inputIndex,FETCH_MAX);
+      inputIndex += FETCH_MAX;
     }
-    loadMultipleInternal(rval,outputIndex,idsToDo,readOnliesToDo);
+    loadMultipleInternal(rval,outputIndex,ids,readOnlies,inputIndex,ids.length-inputIndex);
     return rval;
   }
 
-  protected int loadMultipleInternal(IJobDescription[] rval, int outputIndex, List<Long>
ids, List<Boolean> readOnlies)
+  protected int loadMultipleInternal(IJobDescription[] rval, int outputIndex, Long[] ids,
boolean[] readOnlies, int inputIndex, int length)
     throws ManifoldCFException
   {
     // Build description objects
-    JobObjectDescription[] objectDescriptions = new JobObjectDescription[ids.size()];
+    JobObjectDescription[] objectDescriptions = new JobObjectDescription[length];
     StringSetBuffer ssb = new StringSetBuffer();
-    for (int i = 0; i < ids.size(); i++)
+    for (int i = 0; i < length; i++)
     {
-      Long id = ids.get(i);
+      Long id = ids[inputIndex + i];
       ssb.clear();
       ssb.add(getJobIDKey(id));
       objectDescriptions[i] = new JobObjectDescription(id,new StringSet(ssb));
@@ -860,7 +853,7 @@ public class Jobs extends org.apache.man
 
     JobObjectExecutor exec = new JobObjectExecutor(this,objectDescriptions);
     cacheManager.findObjectsAndExecute(objectDescriptions,null,exec,getTransactionID());
-    IJobDescription[] results = exec.getResults(readOnlies);
+    IJobDescription[] results = exec.getResults(readOnlies,inputIndex);
     for (IJobDescription result : results)
     {
       rval[outputIndex++] = result;
@@ -3616,14 +3609,14 @@ public class Jobs extends org.apache.man
     /** Get the result.
     *@return the looked-up or read cached instances.
     */
-    public JobDescription[] getResults(List<Boolean> readOnlies)
+    public JobDescription[] getResults(boolean[] readOnlies, int inputIndex)
     {
       JobDescription[] rval = new JobDescription[returnValues.length];
       for (int i = 0; i < rval.length; i++)
       {
         JobDescription jd = returnValues[i];
         if (jd != null)
-          rval[i] = jd.duplicate(readOnlies.get(i));
+          rval[i] = jd.duplicate(readOnlies[inputIndex + i]);
         else
           rval[i] = null;
       }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1642163&r1=1642162&r2=1642163&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
Thu Nov 27 14:48:39 2014
@@ -311,31 +311,27 @@ public class RepositoryConnectionManager
     IRepositoryConnection[] rval = new IRepositoryConnection[names.length];
     if (names.length == 0)
       return rval;
-    List<String> fetchNames = new ArrayList<String>();
+    int inputIndex = 0;
     int outputIndex = 0;
-    for (String name : names)
+    while (names.length - inputIndex > FETCH_MAX)
     {
-      if (fetchNames.size() == FETCH_MAX)
-      {
-        outputIndex = loadMultipleInternal(rval,outputIndex,fetchNames);
-        fetchNames.clear();
-      }
-      fetchNames.add(name);
+      outputIndex = loadMultipleInternal(rval,outputIndex,names,inputIndex,FETCH_MAX);
+      inputIndex += FETCH_MAX;
     }
-    loadMultipleInternal(rval,outputIndex,fetchNames);
+    loadMultipleInternal(rval,outputIndex,names,inputIndex,names.length-inputIndex);
     return rval;
   }
   
-  protected int loadMultipleInternal(IRepositoryConnection[] rval, int outputIndex, List<String>
fetchNames)
+  protected int loadMultipleInternal(IRepositoryConnection[] rval, int outputIndex, String[]
fetchNames, int inputIndex, int length)
     throws ManifoldCFException
   {
     // Build description objects
-    RepositoryConnectionDescription[] objectDescriptions = new RepositoryConnectionDescription[fetchNames.size()];
+    RepositoryConnectionDescription[] objectDescriptions = new RepositoryConnectionDescription[length];
     StringSetBuffer ssb = new StringSetBuffer();
-    for (int i = 0; i < fetchNames.size(); i++)
+    for (int i = 0; i < length; i++)
     {
       ssb.clear();
-      String name = fetchNames.get(i);
+      String name = fetchNames[inputIndex + i];
       ssb.add(getRepositoryConnectionKey(name));
       objectDescriptions[i] = new RepositoryConnectionDescription(name,new StringSet(ssb));
     }



Mime
View raw message