incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1182635 - in /incubator/lcf/branches/CONNECTORS-256: connectors/wiki/ tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/
Date Thu, 13 Oct 2011 02:07:39 GMT
Author: kwright
Date: Thu Oct 13 02:07:39 2011
New Revision: 1182635

URL: http://svn.apache.org/viewvc?rev=1182635&view=rev
Log:
More work on tests.  Include test that does a basic crawl.

Added:
    incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
  (with props)
Modified:
    incubator/lcf/branches/CONNECTORS-256/connectors/wiki/   (props changed)
    incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java
    incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java

Propchange: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Thu Oct 13 02:07:39 2011
@@ -1,3 +1,4 @@
 build
 dist
 lib
+test-output

Modified: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java?rev=1182635&r1=1182634&r2=1182635&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java
(original)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/BaseDerby.java
Thu Oct 13 02:07:39 2011
@@ -46,7 +46,8 @@ public class BaseDerby extends org.apach
   
   protected DaemonThread daemonThread = null;
   protected Server server = null;
-
+  protected MockWikiService wikiService = null;
+  
   protected String[] getConnectorNames()
   {
     return new String[]{"Wiki Connector"};
@@ -67,6 +68,24 @@ public class BaseDerby extends org.apach
     return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
   }
   
+  // Setup and teardown the mock wiki service
+  
+  @Before
+  public void createWikiService()
+    throws Exception
+  {
+    wikiService = new MockWikiService(getClass());
+    wikiService.start();
+  }
+  
+  @After
+  public void shutdownWikiService()
+    throws Exception
+  {
+    if (wikiService != null)
+      wikiService.stop();
+  }
+  
   // API support
   
   // These methods allow communication with the ManifoldCF api webapp, via the locally-instantiated
jetty

Modified: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java?rev=1182635&r1=1182634&r2=1182635&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
(original)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/MockWikiService.java
Thu Oct 13 02:07:39 2011
@@ -56,9 +56,10 @@ public class MockWikiService
   public void setResources(Map<String,String> checkResources,
     Map<String,String> listResources,
     Map<String,String> timestampQueryResources,
+    Map<String,String> urlQueryResources,
     Map<String,String> docInfoQueryResources)
   {
-    servlet.setResources(checkResources,listResources,timestampQueryResources,docInfoQueryResources);
+    servlet.setResources(checkResources,listResources,timestampQueryResources,urlQueryResources,docInfoQueryResources);
   }
   
   protected static String sortStuff(String input)
@@ -82,6 +83,7 @@ public class MockWikiService
     protected Map<String,String> checkResources = null;
     protected Map<String,String> listResources = null;
     protected Map<String,String> timestampQueryResources = null;
+    protected Map<String,String> urlQueryResources = null;
     protected Map<String,String> docInfoQueryResources = null;
     
     public WikiAPIServlet(Class theResourceClass)
@@ -92,11 +94,13 @@ public class MockWikiService
     public void setResources(Map<String,String> checkResources,
       Map<String,String> listResources,
       Map<String,String> timestampQueryResources,
+      Map<String,String> urlQueryResources,
       Map<String,String> docInfoQueryResources)
     {
       this.checkResources = checkResources;
       this.listResources = listResources;
       this.timestampQueryResources = timestampQueryResources;
+      this.urlQueryResources = urlQueryResources;
       this.docInfoQueryResources = docInfoQueryResources;
     }
     
@@ -121,32 +125,44 @@ public class MockWikiService
           throw new IOException("Action parameter incorrect: "+action);
         String prop = req.getParameter("prop");
         String pageIds = req.getParameter("pageids");
-        String rvprop = req.getParameter("rvprop");
-        String inprop = req.getParameter("inprop");
-        if (rvprop != null && rvprop.equals("timestamp"))
+        if (prop != null && prop.equals("revisions"))
         {
-          // Version query
-          if (prop == null || !prop.equals("revisions"))
-            throw new IOException("prop parameter incorrect for version query: "+prop);
-          if (pageIds == null)
-            throw new IOException("missing pageids parameter, required for version query");
+          String rvprop = req.getParameter("rvprop");
+          if (rvprop != null && rvprop.equals("timestamp"))
+          {
+            // Version query
+            if (pageIds == null)
+              throw new IOException("missing pageids parameter, required for version query");
+            // Sort the pageIds
+            pageIds = sortStuff(pageIds);
+            resourceName = timestampQueryResources.get(pageIds);
+          }
+          else if (rvprop != null && rvprop.equals("user|comment|content"))
+          {
+            // Doc info query
+            if (pageIds == null)
+              throw new IOException("missing pageids parameter, required for docinfo query");
+            if (pageIds.indexOf("|") != -1)
+              throw new IOException("cannot do more than one docinfo request at once");
+            resourceName = docInfoQueryResources.get(pageIds);
+          }
+          else
+            throw new IOException("rvprop parameter missing or incorrect: "+rvprop);
+        }
+        else if (prop != null && prop.equals("info"))
+        {
+          String inprop = req.getParameter("inprop");
+          if (inprop == null || !inprop.equals("url"))
+            throw new IOException("inprop parameter missing or incorrect: "+inprop);
+          // url query
+            if (pageIds == null)
+              throw new IOException("missing pageids parameter, required for url query");
           // Sort the pageIds
           pageIds = sortStuff(pageIds);
-          resourceName = timestampQueryResources.get(pageIds);
-        }
-	else if (rvprop != null && rvprop.equals("user|comment|content"))
-	{
-          // Doc info query
-          if (prop == null || !prop.equals("revisions"))
-            throw new IOException("prop parameter incorrect for docinfo query: "+prop);
-          if (pageIds == null)
-            throw new IOException("missing pageids parameter, required for docinfo query");
-          if (pageIds.indexOf("|") != -1)
-            throw new IOException("cannot do more than one docinfo request at once");
-          resourceName = docInfoQueryResources.get(pageIds);
+          resourceName = urlQueryResources.get(pageIds);
         }
         else
-          throw new IOException("rvprop parameter missing or incorrect: "+rvprop);
+          throw new IOException("Invalid value for prop parameter: "+prop);
       }
       else if (list != null)
       {

Added: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java?rev=1182635&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
(added)
+++ incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
Thu Oct 13 02:07:39 2011
@@ -0,0 +1,230 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.wiki_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import org.apache.manifoldcf.crawler.connectors.wiki.WikiConfig;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class SanityDerbyTest extends BaseDerby
+{
+  protected static Map<String,String> initialCheckResources;
+  protected static Map<String,String> initialListResources;
+  protected static Map<String,String> initialTimestampQueryResources;
+  protected static Map<String,String> initialURLQueryResources;
+  protected static Map<String,String> initialDocInfoQueryResources;
+  
+  static
+  {
+    initialCheckResources.put("","list_one.xml");
+    
+    initialListResources.put("","list_full.xml");
+    initialListResources.put("Kre Mbaye","list_full_last.xml");
+    
+    initialTimestampQueryResources.put("14773725|19219017|19319577|19839654|30955295","get_timestamps.xml");
+    
+    initialURLQueryResources.put("14773725|19219017|19319577|19839654|30955295","get_urls.xml");
+    
+    initialDocInfoQueryResources.put("14773725","14773725.xml");
+    initialDocInfoQueryResources.put("19219017","19219017.xml");
+    initialDocInfoQueryResources.put("19319577","19319577.xml");
+    initialDocInfoQueryResources.put("19839654","19839654.xml");
+    initialDocInfoQueryResources.put("30955295","30955295.xml");
+    
+  }
+  
+  @Test
+  public void sanityCheck()
+    throws Exception
+  {
+    try
+    {
+      // Hey, we were able to install the file system connector etc.
+      // Now, create a local test job and run it.
+      IThreadContext tc = ThreadContextFactory.make();
+      
+      // Create a basic file system connection, and save it.
+      IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+      IRepositoryConnection conn = mgr.create();
+      conn.setName("Wiki Connection");
+      conn.setDescription("Wiki Connection");
+      conn.setClassName("org.apache.manifoldcf.crawler.connectors.wiki.WikiConnector");
+      conn.setMaxConnections(10);
+      ConfigParams cp = conn.getConfigParams();
+      cp.setParameter(WikiConfig.PARAM_PROTOCOL,"http");
+      cp.setParameter(WikiConfig.PARAM_SERVER,"localhost");
+      cp.setParameter(WikiConfig.PARAM_PORT,"8089");
+      cp.setParameter(WikiConfig.PARAM_PATH,"/w");
+      
+      // Now, save
+      mgr.save(conn);
+      
+      // Create a basic null output connection, and save it.
+      IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+      IOutputConnection outputConn = outputMgr.create();
+      outputConn.setName("Null Connection");
+      outputConn.setDescription("Null Connection");
+      outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+      outputConn.setMaxConnections(100);
+      // Now, save
+      outputMgr.save(outputConn);
+
+      // Create a job.
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      IJobDescription job = jobManager.createJob();
+      job.setDescription("Test Job");
+      job.setConnectionName("Wiki Connection");
+      job.setOutputConnectionName("Null Connection");
+      job.setType(job.TYPE_SPECIFIED);
+      job.setStartMethod(job.START_DISABLE);
+      job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      
+      // Now, set up the document specification.
+      // Right now we don't need any...
+      DocumentSpecification ds = job.getSpecification();
+      
+      // Set up the output specification.
+      OutputSpecification os = job.getOutputSpecification();
+      // Null output connections have no output specification, so this is a no-op.
+      
+      // Save the job.
+      jobManager.save(job);
+
+      // Initialize the mock service
+      wikiService.setResources(initialCheckResources,
+        initialListResources,
+        initialTimestampQueryResources,
+        initialURLQueryResources,
+        initialDocInfoQueryResources);
+        
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      JobStatus status = jobManager.getStatus(job.getID());
+      // The test data area has 3 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 5)
+        throw new ManifoldCFException("Wrong number of documents processed - expected 5,
saw "+new Long(status.getDocumentsProcessed()).toString());
+      
+      /*
+      // Add a file and recrawl
+      createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 6)
+        throw new ManifoldCFException("Wrong number of documents processed after add - expected
6, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+      // Change a file, and recrawl
+      changeFile(new File("testdata/test1.txt"),"Modified contents");
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 6)
+        throw new ManifoldCFException("Wrong number of documents processed after change -
expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+      // We also need to make sure the new document was indexed.  Have to think about how
to do this though.
+      // MHL
+      
+      // Delete a file, and recrawl
+      removeFile(new File("testdata/test2.txt"));
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 3 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 5)
+        throw new ManifoldCFException("Wrong number of documents processed after delete -
expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
+      */
+      
+      // Now, delete the job.
+      jobManager.deleteJob(job.getID());
+      waitJobDeleted(jobManager,job.getID(),120000L);
+      
+      // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
lying around.
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        throw new ManifoldCFException("No such job: '"+jobID+"'");
+      int statusValue = status.getStatus();
+      switch (statusValue)
+      {
+        case JobStatus.JOBSTATUS_NOTYETRUN:
+          throw new ManifoldCFException("Job was never started.");
+        case JobStatus.JOBSTATUS_COMPLETED:
+          break;
+        case JobStatus.JOBSTATUS_ERROR:
+          throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+        default:
+          ManifoldCF.sleep(1000L);
+          continue;
+      }
+      return;
+    }
+    throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new
Long(maxTime).toString()+" milliseconds");
+  }
+  
+  protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        return;
+      ManifoldCF.sleep(1000L);
+    }
+    throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new
Long(maxTime).toString()+" milliseconds");
+  }
+    
+
+}

Propchange: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/branches/CONNECTORS-256/tests/wiki/src/test/java/org/apache/manifoldcf/wiki_tests/SanityDerbyTest.java
------------------------------------------------------------------------------
    svn:keywords = Id



Mime
View raw message