manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1548081 [3/3] - in /manifoldcf/trunk: ./ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/ framework/agents/src/main/java/org/apache/manif...
Date Thu, 05 Dec 2013 10:31:02 GMT
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Thu Dec  5 10:31:01 2013
@@ -105,6 +105,15 @@ public class ManifoldCF extends org.apac
   
   public static void localCleanup(IThreadContext tc)
   {
+    try
+    {
+      RepositoryConnectorPoolFactory.make(tc).closeAllConnectors();
+    }
+    catch (ManifoldCFException e)
+    {
+      if (Logging.root != null)
+        Logging.root.warn("Exception tossed on repository connector pool cleanup: "+e.getMessage(),e);
+    }
   }
   
   /** Create system database using superuser properties from properties.xml.
@@ -948,8 +957,7 @@ public class ManifoldCF extends org.apac
         break;
 
       // Calculate new priorities for all these documents
-      writeDocumentPriorities(threadContext,connectionManager,jobManager,docs,connectionMap,jobDescriptionMap,
-        rt,updateTime);
+      writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,rt,updateTime);
 
       Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed
documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
     }
@@ -959,12 +967,15 @@ public class ManifoldCF extends org.apac
   
   /** Write a set of document priorities, based on the current queue tracker.
   */
-  public static void writeDocumentPriorities(IThreadContext threadContext, IRepositoryConnectionManager
mgr,
-    IJobManager jobManager, DocumentDescription[] descs,
+  public static void writeDocumentPriorities(IThreadContext threadContext, DocumentDescription[]
descs,
     Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription>
jobDescriptionMap,
     ReprioritizationTracker rt, long currentTime)
     throws ManifoldCFException
   {
+    IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+    IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(threadContext);
+    IJobManager jobManager = JobManagerFactory.make(threadContext);
+    
     if (Logging.scheduling.isDebugEnabled())
       Logging.scheduling.debug("Reprioritizing "+Integer.toString(descs.length)+" documents");
 
@@ -992,10 +1003,7 @@ public class ManifoldCF extends org.apac
 
       String[] binNames;
       // Grab a connector handle
-      IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
-        connection.getClassName(),
-        connection.getConfigParams(),
-        connection.getMaxConnections());
+      IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
       try
       {
         if (connector == null)
@@ -1006,7 +1014,7 @@ public class ManifoldCF extends org.apac
       }
       finally
       {
-        RepositoryConnectorFactory.release(connector);
+        repositoryConnectorPool.release(connector);
       }
 
       priorities[i] = new PriorityCalculator(rt,connection,binNames);
@@ -1217,6 +1225,7 @@ public class ManifoldCF extends org.apac
   {
     try
     {
+      IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(tc);
       IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
       IOutputConnection connection = connectionManager.load(connectionName);
       if (connection == null)
@@ -1227,7 +1236,7 @@ public class ManifoldCF extends org.apac
           
       String results;
       // Grab a connection handle, and call the test method
-      IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+      IOutputConnector connector = outputConnectorPool.grab(connection);
       try
       {
         results = connector.check();
@@ -1238,7 +1247,7 @@ public class ManifoldCF extends org.apac
       }
       finally
       {
-        OutputConnectorFactory.release(connector);
+        outputConnectorPool.release(connector);
       }
           
       ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1258,6 +1267,7 @@ public class ManifoldCF extends org.apac
   {
     try
     {
+      IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(tc);
       IAuthorityConnectionManager connectionManager = AuthorityConnectionManagerFactory.make(tc);
       IAuthorityConnection connection = connectionManager.load(connectionName);
       if (connection == null)
@@ -1268,7 +1278,7 @@ public class ManifoldCF extends org.apac
           
       String results;
       // Grab a connection handle, and call the test method
-      IAuthorityConnector connector = AuthorityConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+      IAuthorityConnector connector = authorityConnectorPool.grab(connection);
       try
       {
         results = connector.check();
@@ -1279,7 +1289,7 @@ public class ManifoldCF extends org.apac
       }
       finally
       {
-        AuthorityConnectorFactory.release(connector);
+        authorityConnectorPool.release(connector);
       }
           
       ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1299,6 +1309,7 @@ public class ManifoldCF extends org.apac
   {
     try
     {
+      IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(tc);
       IMappingConnectionManager connectionManager = MappingConnectionManagerFactory.make(tc);
       IMappingConnection connection = connectionManager.load(connectionName);
       if (connection == null)
@@ -1309,7 +1320,7 @@ public class ManifoldCF extends org.apac
           
       String results;
       // Grab a connection handle, and call the test method
-      IMappingConnector connector = MappingConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+      IMappingConnector connector = mappingConnectorPool.grab(connection);
       try
       {
         results = connector.check();
@@ -1320,7 +1331,7 @@ public class ManifoldCF extends org.apac
       }
       finally
       {
-        MappingConnectorFactory.release(connector);
+        mappingConnectorPool.release(connector);
       }
           
       ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1340,6 +1351,7 @@ public class ManifoldCF extends org.apac
   {
     try
     {
+      IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(tc);
       IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
       IRepositoryConnection connection = connectionManager.load(connectionName);
       if (connection == null)
@@ -1350,7 +1362,7 @@ public class ManifoldCF extends org.apac
           
       String results;
       // Grab a connection handle, and call the test method
-      IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+      IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
       try
       {
         results = connector.check();
@@ -1361,7 +1373,7 @@ public class ManifoldCF extends org.apac
       }
       finally
       {
-        RepositoryConnectorFactory.release(connector);
+        repositoryConnectorPool.release(connector);
       }
           
       ConfigurationNode response = new ConfigurationNode(API_CHECKRESULTNODE);
@@ -1382,6 +1394,7 @@ public class ManifoldCF extends org.apac
   {
     try
     {
+      IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(tc);
       IOutputConnectionManager connectionManager = OutputConnectionManagerFactory.make(tc);
       IOutputConnection connection = connectionManager.load(connectionName);
       if (connection == null)
@@ -1391,14 +1404,14 @@ public class ManifoldCF extends org.apac
       }
 
       // Grab a connection handle, and call the test method
-      IOutputConnector connector = OutputConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+      IOutputConnector connector = outputConnectorPool.grab(connection);
       try
       {
         return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
       }
       finally
       {
-        OutputConnectorFactory.release(connector);
+        outputConnectorPool.release(connector);
       }
     }
     catch (ManifoldCFException e)
@@ -1414,6 +1427,7 @@ public class ManifoldCF extends org.apac
   {
     try
     {
+      IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(tc);
       IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(tc);
       IRepositoryConnection connection = connectionManager.load(connectionName);
       if (connection == null)
@@ -1423,14 +1437,14 @@ public class ManifoldCF extends org.apac
       }
 
       // Grab a connection handle, and call the test method
-      IRepositoryConnector connector = RepositoryConnectorFactory.grab(tc,connection.getClassName(),connection.getConfigParams(),connection.getMaxConnections());
+      IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
       try
       {
         return connector.requestInfo(output,command)?READRESULT_FOUND:READRESULT_NOTFOUND;
       }
       finally
       {
-        RepositoryConnectorFactory.release(connector);
+        repositoryConnectorPool.release(connector);
       }
     }
     catch (ManifoldCFException e)

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
Thu Dec  5 10:31:01 2013
@@ -68,11 +68,8 @@ public class SeedingThread extends Threa
       IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
       ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
 
-      IDBInterface database = DBInterfaceFactory.make(threadContext,
-        ManifoldCF.getMasterDatabaseName(),
-        ManifoldCF.getMasterDatabaseUsername(),
-        ManifoldCF.getMasterDatabasePassword());
-
+      IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+      
       String[] identifiers = new String[MAX_COUNT];
       // Loop
       while (true)
@@ -126,17 +123,12 @@ public class SeedingThread extends Threa
                 int hopcountMethod = jobDescription.getHopcountMode();
 
                 IRepositoryConnection connection = connectionMgr.load(jobDescription.getConnectionName());
-                IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
-                  connection.getClassName(),
-                  connection.getConfigParams(),
-                  connection.getMaxConnections());
+                IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
                 // Null will come back if the connector instance could not be obtained, so
just skip in that case.
                 if (connector == null)
                   continue;
                 try
                 {
-
-
                   // Get the number of link types.
                   String[] legalLinkTypes = connector.getRelationshipTypes();
 
@@ -174,7 +166,7 @@ public class SeedingThread extends Threa
                 }
                 finally
                 {
-                  RepositoryConnectorFactory.release(connector);
+                  repositoryConnectorPool.release(connector);
                 }
 
 

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
Thu Dec  5 10:31:01 2013
@@ -126,7 +126,7 @@ public class SetPriorityThread extends T
             DocumentDescription desc = blockingDocuments.getBlockingDocument();
             if (desc != null)
             {
-              ManifoldCF.writeDocumentPriorities(threadContext,mgr,jobManager,
+              ManifoldCF.writeDocumentPriorities(threadContext,
                 new DocumentDescription[]{desc},connectionMap,jobDescriptionMap,rt,currentTime);
               processedCount++;
               continue;

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
Thu Dec  5 10:31:01 2013
@@ -62,11 +62,8 @@ public class StartupThread extends Threa
       IRepositoryConnectionManager connectionMgr = RepositoryConnectionManagerFactory.make(threadContext);
       ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
 
-      IDBInterface database = DBInterfaceFactory.make(threadContext,
-        ManifoldCF.getMasterDatabaseName(),
-        ManifoldCF.getMasterDatabaseUsername(),
-        ManifoldCF.getMasterDatabasePassword());
-
+      IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+      
       // Loop
       while (true)
       {
@@ -117,22 +114,19 @@ public class StartupThread extends Threa
                 int hopcountMethod = jobDescription.getHopcountMode();
 
                 IRepositoryConnection connection = connectionMgr.load(jobDescription.getConnectionName());
-                IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
-                  connection.getClassName(),
-                  connection.getConfigParams(),
-                  connection.getMaxConnections());
+                IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
 
                 // If the attempt to grab a connector instance failed, don't start the job,
of course.
                 if (connector == null)
                   continue;
 
-                // Only now record the fact that we are trying to start the job.
-                connectionMgr.recordHistory(jobDescription.getConnectionName(),
-                  null,connectionMgr.ACTIVITY_JOBSTART,null,
-                  jobID.toString()+"("+jobDescription.getDescription()+")",null,null,null);
-
                 try
                 {
+                  // Only now record the fact that we are trying to start the job.
+                  connectionMgr.recordHistory(jobDescription.getConnectionName(),
+                    null,connectionMgr.ACTIVITY_JOBSTART,null,
+                    jobID.toString()+"("+jobDescription.getDescription()+")",null,null,null);
+
                   int model = connector.getConnectorModel();
                   // Get the number of link types.
                   String[] legalLinkTypes = connector.getRelationshipTypes();
@@ -175,7 +169,7 @@ public class StartupThread extends Threa
                 }
                 finally
                 {
-                  RepositoryConnectorFactory.release(connector);
+                  repositoryConnectorPool.release(connector);
                 }
 
                 // Start this job!

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StufferThread.java
Thu Dec  5 10:31:01 2013
@@ -89,6 +89,8 @@ public class StufferThread extends Threa
       IJobManager jobManager = JobManagerFactory.make(threadContext);
       ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
 
+      IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+      
       Logging.threads.debug("Stuffer thread: Low water mark is "+Integer.toString(lowWaterMark)+";
amount per stuffing is "+Integer.toString(stuffAmt));
 
       // This is used to adjust the number of records returned for jobs
@@ -250,10 +252,7 @@ public class StufferThread extends Threa
             try
             {
               // Grab a connector handle
-              IRepositoryConnector connector = RepositoryConnectorFactory.grab(threadContext,
-                connection.getClassName(),
-                connection.getConfigParams(),
-                connection.getMaxConnections());
+              IRepositoryConnector connector = repositoryConnectorPool.grab(connection);
               if (connector == null)
               {
                 maxDocuments = 1;
@@ -270,7 +269,7 @@ public class StufferThread extends Threa
                 }
                 finally
                 {
-                  RepositoryConnectorFactory.release(connector);
+                  repositoryConnectorPool.release(connector);
                 }
               }
             }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1548081&r1=1548080&r2=1548081&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Thu Dec  5 10:31:01 2013
@@ -79,6 +79,8 @@ public class WorkerThread extends Thread
       IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(threadContext);
       ReprioritizationTracker rt = new ReprioritizationTracker(threadContext);
 
+      IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
+      
       List<DocumentToProcess> fetchList = new ArrayList<DocumentToProcess>();
       Map<String,String> versionMap = new HashMap<String,String>();
       List<QueuedDocument> finishList = new ArrayList<QueuedDocument>();
@@ -257,10 +259,7 @@ public class WorkerThread extends Thread
               IRepositoryConnector connector = null;
               if (activeDocuments.size() > 0 || hopcountremoveList.size() > 0)
               {
-                connector = RepositoryConnectorFactory.grab(threadContext,
-                  connection.getClassName(),
-                  connection.getConfigParams(),
-                  connection.getMaxConnections());
+                connector = repositoryConnectorPool.grab(connection);
 
                 // If we wind up with a null here, it means that a document got queued for
a connector which is now gone.
                 // Basically, what we want to do in that case is to treat this kind of like
a service interruption - the document
@@ -832,7 +831,7 @@ public class WorkerThread extends Thread
                 }
                 finally
                 {
-                  RepositoryConnectorFactory.release(connector);
+                  repositoryConnectorPool.release(connector);
                 }
               
               }



Mime
View raw message