manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1214847 - in /incubator/lcf/branches/CONNECTORS-313/framework: jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ pull-agent/src/main/java/org/apache/manifoldcf/crawler/ pull-agent/src/main/java/org/apache/manifoldcf/crawler/int...
Date Thu, 15 Dec 2011 16:34:25 GMT
Author: kwright
Date: Thu Dec 15 16:34:24 2011
New Revision: 1214847

URL: http://svn.apache.org/viewvc?rev=1214847&view=rev
Log:
Move the new code into ManifoldCF class, where it naturally belongs

Added:
    incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/Connectors.java
      - copied, changed from r1214737, incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/Connectors.java
Removed:
    incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/Connectors.java
    incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/RegisterConnectors.java
Modified:
    incubator/lcf/branches/CONNECTORS-313/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
    incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/InitializeAndRegister.java
    incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java

Modified: incubator/lcf/branches/CONNECTORS-313/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-313/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java?rev=1214847&r1=1214846&r2=1214847&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-313/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
(original)
+++ incubator/lcf/branches/CONNECTORS-313/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
Thu Dec 15 16:34:24 2011
@@ -19,7 +19,7 @@ package org.apache.manifoldcf.jettyrunne
 
 import java.io.*;
 import org.apache.manifoldcf.core.interfaces.*;
-import org.apache.manifoldcf.agents.system.*;
+import org.apache.manifoldcf.crawler.system.*;
 import org.apache.manifoldcf.crawler.*;
 
 import java.io.IOException;
@@ -194,30 +194,18 @@ public class ManifoldCFJettyRunner
       ManifoldCF.initializeEnvironment();
       IThreadContext tc = ThreadContextFactory.make();
 
-      // Clear the shutdown signal
-      RegisterConnectors.clearShutdownSignal(tc);
-      
-      // Get the specified superuser name and password, in case this isn't Derby we're using
-      String superuserName = ManifoldCF.getProperty(RegisterConnectors.databaseSuperuserName);
-      if (superuserName == null)
-        superuserName = "";
-      String superuserPassword = ManifoldCF.getProperty(RegisterConnectors.databaseSuperuserPassword);
-      if (superuserPassword == null)
-        superuserPassword = "";
+      // Clear the agents shutdown signal.
+      ILockManager lockManager = LockManagerFactory.make(tc);
+      lockManager.clearGlobalFlag(agentShutdownSignal);
       
       // Do the basic initialization of the database and its schema
-      RegisterConnectors.createDatabaseAndSchema(tc,superuserName,superuserPassword);
-      RegisterConnectors.registerAgents(tc);
+      ManifoldCF.createSystemDatabase(tc);
       
-        // Read connectors configuration file (to figure out what we need to register)
-      File connectorConfigFile = ManifoldCF.getFileProperty(RegisterConnectors.connectorsConfigurationFile);
-      Connectors c = RegisterConnectors.readConnectorDeclarations(connectorConfigFile);
+      ManifoldCF.installTables(tc);
       
-      // Unregister all connectors.
-      RegisterConnectors.unregisterAllConnectors(tc);
-
-      // Register connections specified by connectors.xml
-      RegisterConnectors.registerConnectors(tc,c);
+      org.apache.manifoldcf.crawler.system.ManifoldCF.registerThisAgent(tc);
+      
+      ManifoldCF.reregisterAllConnectors(tc);
       
       System.err.println("Starting jetty...");
       

Modified: incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/InitializeAndRegister.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/InitializeAndRegister.java?rev=1214847&r1=1214846&r2=1214847&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/InitializeAndRegister.java
(original)
+++ incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/InitializeAndRegister.java
Thu Dec 15 16:34:24 2011
@@ -20,7 +20,7 @@ package org.apache.manifoldcf.crawler;
 
 import java.io.File;
 
-import org.apache.manifoldcf.agents.system.ManifoldCF;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
 import org.apache.manifoldcf.core.interfaces.*;
 
 public class InitializeAndRegister
@@ -33,27 +33,14 @@ public class InitializeAndRegister
     
     protected void doExecute(IThreadContext tc) throws ManifoldCFException 
     {
-      // Get the specified superuser name and password, in case this isn't Derby we're using
-      String superuserName = ManifoldCF.getProperty(RegisterConnectors.databaseSuperuserName);
-      if (superuserName == null)
-        superuserName = "";
-        String superuserPassword = ManifoldCF.getProperty(RegisterConnectors.databaseSuperuserPassword);
-      if (superuserPassword == null)
-	superuserPassword = "";
-	      
       // Do the basic initialization of the database and its schema
-      RegisterConnectors.createDatabaseAndSchema(tc,superuserName,superuserPassword);
-      RegisterConnectors.registerAgents(tc);
+      ManifoldCF.createSystemDatabase(tc);
       
-      // Read connectors configuration file (to figure out what we need to register)
-      File connectorConfigFile = ManifoldCF.getFileProperty(RegisterConnectors.connectorsConfigurationFile);
-      Connectors c = RegisterConnectors.readConnectorDeclarations(connectorConfigFile);
+      ManifoldCF.installTables(tc);
 
-      // Unregister all connectors, prior to reregistering.  This allows connectors.xml to
control deletions too.
-      RegisterConnectors.unregisterAllConnectors(tc);
+      org.apache.manifoldcf.crawler.system.ManifoldCF.registerThisAgent(tc);
       
-      // Register connections specified by connectors.xml
-      RegisterConnectors.registerConnectors(tc,c);	
+      ManifoldCF.reregisterAllConnectors(tc);
     }
     
     public static void main(String[] args)

Copied: incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/Connectors.java
(from r1214737, incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/Connectors.java)
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/Connectors.java?p2=incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/Connectors.java&p1=incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/Connectors.java&r1=1214737&r2=1214847&rev=1214847&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/Connectors.java
(original)
+++ incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/Connectors.java
Thu Dec 15 16:34:24 2011
@@ -16,7 +16,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.manifoldcf.crawler;
+package org.apache.manifoldcf.crawler.interfaces;
 
 import org.apache.manifoldcf.core.interfaces.*;
 import java.util.*;

Modified: incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1214847&r1=1214846&r2=1214847&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
(original)
+++ incubator/lcf/branches/CONNECTORS-313/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Thu Dec 15 16:34:24 2011
@@ -75,12 +75,16 @@ public class ManifoldCF extends org.apac
   // Factor in amount to stuff
   protected static float stuffAmtFactor = 0.5f;
 
+  // Properties
   protected static final String workerThreadCountProperty = "org.apache.manifoldcf.crawler.threads";
   protected static final String deleteThreadCountProperty = "org.apache.manifoldcf.crawler.deletethreads";
   protected static final String cleanupThreadCountProperty = "org.apache.manifoldcf.crawler.cleanupthreads";
   protected static final String expireThreadCountProperty = "org.apache.manifoldcf.crawler.expirethreads";
   protected static final String lowWaterFactorProperty = "org.apache.manifoldcf.crawler.lowwaterfactor";
   protected static final String stuffAmtFactorProperty = "org.apache.manifoldcf.crawler.stuffamountfactor";
+  protected static final String connectorsConfigurationFileProperty = "org.apache.manifoldcf.connectorsconfigurationfile";
+  protected static final String databaseSuperuserNameProperty = "org.apache.manifoldcf.dbsuperusername";
+  protected static final String databaseSuperuserPasswordProperty = "org.apache.manifoldcf.dbsuperuserpassword";
 
   /** This object is used to make sure the initialization sequence is atomic.  Shutdown cannot
occur until the system is in a known state. */
   protected static Integer startupLock = new Integer(0);
@@ -104,6 +108,290 @@ public class ManifoldCF extends org.apac
     }
   }
 
+  /** Create system database using superuser properties from properties.xml.
+  */
+  public static void createSystemDatabase(IThreadContext threadContext)
+    throws ManifoldCFException
+  {
+    // Get the specified superuser name and password, in case this isn't Derby we're using
+    String superuserName = getProperty(databaseSuperuserNameProperty);
+    if (superuserName == null)
+      superuserName = "";
+    String superuserPassword = getProperty(databaseSuperuserPasswordProperty);
+    if (superuserPassword == null)
+      superuserPassword = "";
+    createSystemDatabase(threadContext,superuserName,superuserPassword);
+  }
+  
+  /** Register this agent */
+  public static void registerThisAgent(IThreadContext tc)
+    throws ManifoldCFException
+  {
+    // Register 
+    IAgentManager agentMgr = AgentManagerFactory.make(tc);
+    agentMgr.registerAgent("org.apache.manifoldcf.crawler.system.CrawlerAgent");
+  }
+
+  /** Register or re-register all connectors, based on a connectors.xml file.
+  */
+  public static void reregisterAllConnectors(IThreadContext tc)
+    throws ManifoldCFException
+  {
+    // Read connectors configuration file (to figure out what we need to register)
+    File connectorConfigFile = getFileProperty(connectorsConfigurationFileProperty);
+    Connectors c = readConnectorDeclarations(connectorConfigFile);
+      
+    // Unregister all connectors.
+    unregisterAllConnectors(tc);
+
+    // Register connections specified by connectors.xml
+    registerConnectors(tc,c);
+  }
+  
+  /** Read connectors configuration file.
+  */
+  public static Connectors readConnectorDeclarations(File connectorConfigFile)
+    throws ManifoldCFException
+  {
+    Connectors c = null;
+    if (connectorConfigFile != null)
+    {
+      try
+      {
+        // Open the file, read it, and attempt to do the connector registrations
+        InputStream is = new FileInputStream(connectorConfigFile);
+        try
+        {
+          c = new Connectors(is);
+        }
+        finally
+        {
+          is.close();
+        }
+      }
+      catch (FileNotFoundException e)
+      {
+        throw new ManifoldCFException("Couldn't find connector configuration file: "+e.getMessage(),e);
+      }
+      catch (IOException e)
+      {
+        throw new ManifoldCFException("Error reading connector configuration file: "+e.getMessage(),e);
+      }
+    }
+    return c;
+  }
+
+  /** Unregister all connectors.
+  */
+  public static void unregisterAllConnectors(IThreadContext tc)
+    throws ManifoldCFException
+  {
+    // Grab a database handle, so we can use transactions later.
+    IDBInterface database = DBInterfaceFactory.make(tc,
+      ManifoldCF.getMasterDatabaseName(),
+      ManifoldCF.getMasterDatabaseUsername(),
+      ManifoldCF.getMasterDatabasePassword());
+
+    // Output connectors...
+    {
+      IOutputConnectorManager mgr = OutputConnectorManagerFactory.make(tc);
+      IOutputConnectionManager connManager = OutputConnectionManagerFactory.make(tc);
+      IResultSet classNames = mgr.getConnectors();
+      int i = 0;
+      while (i < classNames.getRowCount())
+      {
+        IResultRow row = classNames.getRow(i++);
+        String className = (String)row.getValue("classname");
+        // Deregistration should be done in a transaction
+        database.beginTransaction();
+        try
+        {
+          // Find the connection names that come with this class
+          String[] connectionNames = connManager.findConnectionsForConnector(className);
+          // For all connection names, notify all agents of the deregistration
+          AgentManagerFactory.noteOutputConnectorDeregistration(tc,connectionNames);
+          // Now that all jobs have been placed into an appropriate state, actually do the
deregistration itself.
+          mgr.unregisterConnector(className);
+        }
+        catch (ManifoldCFException e)
+        {
+          database.signalRollback();
+          throw e;
+        }
+        catch (Error e)
+        {
+          database.signalRollback();
+          throw e;
+        }
+        finally
+        {
+          database.endTransaction();
+        }
+      }
+      System.err.println("Successfully unregistered all output connectors");
+    }
+      
+    // Authority connectors...
+    {
+      IAuthorityConnectorManager mgr = AuthorityConnectorManagerFactory.make(tc);
+      IResultSet classNames = mgr.getConnectors();
+      int i = 0;
+      while (i < classNames.getRowCount())
+      {
+        IResultRow row = classNames.getRow(i++);
+        mgr.unregisterConnector((String)row.getValue("classname"));
+      }
+      System.err.println("Successfully unregistered all authority connectors");
+    }
+      
+    // Repository connectors...
+    {
+      IConnectorManager mgr = ConnectorManagerFactory.make(tc);
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
+      IResultSet classNames = mgr.getConnectors();
+      int i = 0;
+      while (i < classNames.getRowCount())
+      {
+        IResultRow row = classNames.getRow(i++);
+        String className = (String)row.getValue("classname");
+        // Deregistration should be done in a transaction
+        database.beginTransaction();
+        try
+        {
+          // Find the connection names that come with this class
+          String[] connectionNames = connManager.findConnectionsForConnector(className);
+          // For each connection name, modify the jobs to note that the connector is no longer
installed
+          jobManager.noteConnectorDeregistration(connectionNames);
+          // Now that all jobs have been placed into an appropriate state, actually do the
deregistration itself.
+          mgr.unregisterConnector(className);
+        }
+        catch (ManifoldCFException e)
+        {
+          database.signalRollback();
+          throw e;
+        }
+        catch (Error e)
+        {
+          database.signalRollback();
+          throw e;
+        }
+        finally
+        {
+          database.endTransaction();
+        }
+      }
+      System.err.println("Successfully unregistered all repository connectors");
+    }
+  }
+
+  // Connectors configuration file
+  protected static final String NODE_OUTPUTCONNECTOR = "outputconnector";
+  protected static final String NODE_AUTHORITYCONNECTOR = "authorityconnector";
+  protected static final String NODE_REPOSITORYCONNECTOR = "repositoryconnector";
+  protected static final String ATTRIBUTE_NAME = "name";
+  protected static final String ATTRIBUTE_CLASS = "class";
+
+  /** Register all connectors as specified by a Connectors structure, usually read from the
connectors.xml file.
+  */
+  public static void registerConnectors(IThreadContext tc, Connectors c)
+    throws ManifoldCFException
+  {
+    if (c != null)
+    {
+      // Grab a database handle, so we can use transactions later.
+      IDBInterface database = DBInterfaceFactory.make(tc,
+        ManifoldCF.getMasterDatabaseName(),
+        ManifoldCF.getMasterDatabaseUsername(),
+        ManifoldCF.getMasterDatabasePassword());
+        
+      // Other code will go here to discover and register various connectors that exist in
the classpath
+      int i = 0;
+      while (i < c.getChildCount())
+      {
+        ConfigurationNode cn = c.findChild(i++);
+        if (cn.getType().equals(NODE_OUTPUTCONNECTOR))
+        {
+          String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+          String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+          IOutputConnectorManager mgr = OutputConnectorManagerFactory.make(tc);
+          IOutputConnectionManager connManager = OutputConnectionManagerFactory.make(tc);
+          // Registration should be done in a transaction
+          database.beginTransaction();
+          try
+          {
+            // First, register connector
+            mgr.registerConnector(name,className);
+            // Then, signal to all jobs that might depend on this connector that they can
switch state
+            // Find the connection names that come with this class
+            String[] connectionNames = connManager.findConnectionsForConnector(className);
+            // For all connection names, notify all agents of the registration
+            AgentManagerFactory.noteOutputConnectorRegistration(tc,connectionNames);
+          }
+          catch (ManifoldCFException e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          catch (Error e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          finally
+          {
+            database.endTransaction();
+          }
+          System.err.println("Successfully registered output connector '"+className+"'");
+        }
+        else if (cn.getType().equals(NODE_AUTHORITYCONNECTOR))
+        {
+          String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+          String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+          IAuthorityConnectorManager mgr = AuthorityConnectorManagerFactory.make(tc);
+          mgr.registerConnector(name,className);
+          System.err.println("Successfully registered authority connector '"+className+"'");
+        }
+        else if (cn.getType().equals(NODE_REPOSITORYCONNECTOR))
+        {
+          String name = cn.getAttributeValue(ATTRIBUTE_NAME);
+          String className = cn.getAttributeValue(ATTRIBUTE_CLASS);
+          IConnectorManager mgr = ConnectorManagerFactory.make(tc);
+          IJobManager jobManager = JobManagerFactory.make(tc);
+          IRepositoryConnectionManager connManager = RepositoryConnectionManagerFactory.make(tc);
+          // Deregistration should be done in a transaction
+          database.beginTransaction();
+          try
+          {
+            // First, register connector
+            mgr.registerConnector(name,className);
+            // Then, signal to all jobs that might depend on this connector that they can
switch state
+            // Find the connection names that come with this class
+            String[] connectionNames = connManager.findConnectionsForConnector(className);
+            // For each connection name, modify the jobs to note that the connector is now
installed
+            jobManager.noteConnectorRegistration(connectionNames);
+          }
+          catch (ManifoldCFException e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          catch (Error e)
+          {
+            database.signalRollback();
+            throw e;
+          }
+          finally
+          {
+            database.endTransaction();
+          }
+          System.err.println("Successfully registered repository connector '"+className+"'");
+        }
+        else
+          throw new ManifoldCFException("Unrecognized connectors node type '"+cn.getType()+"'");
+      }
+    }
+  }
 
   /** Install all the crawler system tables.
   *@param threadcontext is the thread context.



Mime
View raw message