manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r964422 - in /incubator/lcf/trunk/modules: ./ filesystem-tests/org/apache/lcf/filesystem_tests/ framework/api/org/apache/lcf/api/ framework/core/org/apache/lcf/core/interfaces/ framework/pull-agent/org/apache/lcf/crawler/system/
Date Thu, 15 Jul 2010 13:38:10 GMT
Author: kwright
Date: Thu Jul 15 13:38:10 2010
New Revision: 964422

URL: http://svn.apache.org/viewvc?rev=964422&view=rev
Log:
Add the first small API test, and API test support.  Also, make the Configuration class not
abstract, since that's an impediment to easy usage, and doesn't really buy us anything.

Added:
    incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java
  (with props)
Removed:
    incubator/lcf/trunk/modules/framework/api/org/apache/lcf/api/Argument.java
Modified:
    incubator/lcf/trunk/modules/build.xml
    incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/TestBase.java
    incubator/lcf/trunk/modules/framework/api/org/apache/lcf/api/APIServlet.java
    incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/Configuration.java
    incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/system/LCF.java

Modified: incubator/lcf/trunk/modules/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/build.xml?rev=964422&r1=964421&r2=964422&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/build.xml (original)
+++ incubator/lcf/trunk/modules/build.xml Thu Jul 15 13:38:10 2010
@@ -747,6 +747,7 @@
                 <pathelement location="build/filesystem-tests/classes"/>
             </classpath>
             <test name="org.apache.lcf.filesystem_tests.Sanity" todir="test-output/filesystem"/>
+            <test name="org.apache.lcf.filesystem_tests.APISanity" todir="test-output/filesystem"/>
             <!-- MHL -->
         </junit>
     </target>

Added: incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java?rev=964422&view=auto
==============================================================================
--- incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java
(added)
+++ incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java
Thu Jul 15 13:38:10 2010
@@ -0,0 +1,263 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.lcf.filesystem_tests;
+
+import org.apache.lcf.core.interfaces.*;
+import org.apache.lcf.agents.interfaces.*;
+import org.apache.lcf.crawler.interfaces.*;
+import org.apache.lcf.crawler.system.LCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class APISanity extends TestBase
+{
+  
+  @Before
+  public void createTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+      createDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void removeTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @Test
+  public void sanityCheck()
+    throws Exception
+  {
+    try
+    {
+      // Hey, we were able to install the file system connector etc.
+      // Now, create a local test job and run it.
+      IThreadContext tc = ThreadContextFactory.make();
+      
+      // Create a basic file system connection, and save it.
+      ConfigurationNode connectionObject = new ConfigurationNode("repositoryconnection");
+      ConfigurationNode child;
+      
+      child = new ConfigurationNode("name");
+      child.setValue("File Connection");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+      
+      child = new ConfigurationNode("class_name");
+      child.setValue("org.apache.lcf.crawler.connectors.filesystem.FileConnector");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+      
+      child = new ConfigurationNode("description");
+      child.setValue("File Connection");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+
+      child = new ConfigurationNode("max_connections");
+      child.setValue("100");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+
+      Configuration requestObject = new Configuration();
+      requestObject.addChild(0,connectionObject);
+      
+      Configuration result = performAPIOperationViaNodes("repositoryconnection/save",requestObject);
+      
+      int i = 0;
+      while (i < result.getChildCount())
+      {
+        ConfigurationNode resultNode = result.findChild(i++);
+        if (resultNode.getType() == "error")
+          throw new Exception(resultNode.getValue());
+      }
+      
+      // Create a basic null output connection, and save it.
+      IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+      IOutputConnection outputConn = outputMgr.create();
+      outputConn.setName("Null Connection");
+      outputConn.setDescription("Null Connection");
+      outputConn.setClassName("org.apache.lcf.agents.output.nullconnector.NullConnector");
+      outputConn.setMaxConnections(100);
+      // Now, save
+      outputMgr.save(outputConn);
+
+      // Create a job.
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      IJobDescription job = jobManager.createJob();
+      job.setDescription("Test Job");
+      job.setConnectionName("File Connection");
+      job.setOutputConnectionName("Null Connection");
+      job.setType(job.TYPE_SPECIFIED);
+      job.setStartMethod(job.START_DISABLE);
+      job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      
+      // Now, set up the document specification.
+      DocumentSpecification ds = job.getSpecification();
+      // Crawl everything underneath the 'testdata' area
+      File testDataFile = new File("testdata").getCanonicalFile();
+      if (!testDataFile.exists())
+        throw new LCFException("Test data area not found!  Looking in "+testDataFile.toString());
+      if (!testDataFile.isDirectory())
+        throw new LCFException("Test data area not a directory!  Looking in "+testDataFile.toString());
+      SpecificationNode sn = new SpecificationNode("startpoint");
+      sn.setAttribute("path",testDataFile.toString());
+      SpecificationNode n = new SpecificationNode("include");
+      n.setAttribute("type","file");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      n = new SpecificationNode("include");
+      n.setAttribute("type","directory");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      ds.addChild(ds.getChildCount(),sn);
+      
+      // Set up the output specification.
+      OutputSpecification os = job.getOutputSpecification();
+      // Null output connections have no output specification, so this is a no-op.
+      
+      // Save the job.
+      jobManager.save(job);
+
+      // Create the test data files.
+      createFile(new File("testdata/test1.txt"),"This is a test file");
+      createFile(new File("testdata/test2.txt"),"This is another test file");
+      createDirectory(new File("testdata/testdir"));
+      createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID());
+
+      // Check to be sure we actually processed the right number of documents.
+      JobStatus status = jobManager.getStatus(job.getID());
+      // The test data area has 3 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 5)
+        throw new LCFException("Wrong number of documents processed - expected 5, saw "+new
Long(status.getDocumentsProcessed()).toString());
+      
+      // Add a file and recrawl
+      createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID());
+
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 6)
+        throw new LCFException("Wrong number of documents processed after add - expected
6, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+      // Change a file, and recrawl
+      changeFile(new File("testdata/test1.txt"),"Modified contents");
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID());
+
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 6)
+        throw new LCFException("Wrong number of documents processed after change - expected
6, saw "+new Long(status.getDocumentsProcessed()).toString());
+      // We also need to make sure the new document was indexed.  Have to think about how
to do this though.
+      // MHL
+      
+      // Delete a file, and recrawl
+      removeFile(new File("testdata/test2.txt"));
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID());
+
+      // Check to be sure we actually processed the right number of documents.
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 3 documents and one directory, and we have to count the root
directory too.
+      if (status.getDocumentsProcessed() != 5)
+        throw new LCFException("Wrong number of documents processed after delete - expected
5, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+      // Now, delete the job.
+      jobManager.deleteJob(job.getID());
+      waitJobDeleted(jobManager,job.getID());
+      
+      // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
lying around.
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  protected void waitJobInactive(IJobManager jobManager, Long jobID)
+    throws LCFException, InterruptedException
+  {
+    while (true)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        throw new LCFException("No such job: '"+jobID+"'");
+      int statusValue = status.getStatus();
+      switch (statusValue)
+      {
+        case JobStatus.JOBSTATUS_NOTYETRUN:
+          throw new LCFException("Job was never started.");
+        case JobStatus.JOBSTATUS_COMPLETED:
+          break;
+        case JobStatus.JOBSTATUS_ERROR:
+          throw new LCFException("Job reports error status: "+status.getErrorText());
+        default:
+          LCF.sleep(10000L);
+          continue;
+      }
+      break;
+    }
+  }
+  
+  protected void waitJobDeleted(IJobManager jobManager, Long jobID)
+    throws LCFException, InterruptedException
+  {
+    while (true)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        break;
+      LCF.sleep(10000L);
+    }
+  }
+    
+
+}

Propchange: incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/APISanity.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/TestBase.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/TestBase.java?rev=964422&r1=964421&r2=964422&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/TestBase.java
(original)
+++ incubator/lcf/trunk/modules/filesystem-tests/org/apache/lcf/filesystem_tests/TestBase.java
Thu Jul 15 13:38:10 2010
@@ -27,13 +27,26 @@ import java.io.*;
 import java.util.*;
 import org.junit.*;
 
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.mortbay.log.Logger;
+
+import org.apache.commons.httpclient.*;
+import org.apache.commons.httpclient.methods.*;
+
 /** Tests that run the "agents daemon" should be derived from this */
 public class TestBase extends org.apache.lcf.crawler.tests.TestConnectorBase
 {
   public static final String agentShutdownSignal = "agent-process";
+  public static final int testPort = 8346;
   
   protected DaemonThread daemonThread = null;
-  
+  protected Server server = null;
+
   protected String[] getConnectorNames()
   {
     return new String[]{"File Connector"};
@@ -116,11 +129,73 @@ public class TestBase extends org.apache
     createFile(f,newContents);
   }
   
+  // API support
+  
+  // These methods allow communication with the LCF api webapp, via the locally-instantiated
jetty
+  
+  /** Perform an json API operation.
+  *@param command is the operation.
+  *@param argument is the json argument, or null if none.
+  *@return the json response.
+  */
+  protected String performAPIOperation(String command, String argument)
+    throws Exception
+  {
+    HttpClient client = new HttpClient();
+    HttpMethod method = new GetMethod("http://localhost:"+Integer.toString(testPort)+"/lcf-api/json/"+command+((argument==null)?"":"?object="+
+      java.net.URLEncoder.encode(argument,"utf-8")));
+    int response = client.executeMethod(method);
+    byte[] responseData = method.getResponseBody();
+    String responseString = new String(responseData,"utf-8");
+    if (response != 200)
+      throw new Exception("API http error "+Integer.toString(response)+": "+responseString);
+    // We presume that the data is utf-8, since that's what the API uses throughout.
+    return responseString;
+  }
+  
+  /** Perform a json API operation, using Configuration structures to represent the json.
 This is for testing convenience,
+  * mostly.
+  */
+  protected Configuration performAPIOperationViaNodes(String command, Configuration argument)
+    throws Exception
+  {
+    String argumentJson;
+    if (argument != null)
+      argumentJson = argument.toJSON();
+    else
+      argumentJson = null;
+    
+    String result = performAPIOperation(command,argumentJson);
+    Configuration cfg = new Configuration();
+    cfg.fromJSON(result);
+    return cfg;
+  }
+  
+  // Setup/teardown
+  
   @Before
   public void setUp()
     throws Exception
   {
     super.setUp();
+    // Start jetty
+    server = new Server( testPort );    
+    server.setStopAtShutdown( true );
+    
+    // Initialize the servlets
+    WebAppContext lcfCrawlerUI = new WebAppContext("../../framework/dist/web/war/lcf-crawler-ui.war","/lcf-crawler-ui");
+    // This will cause jetty to ignore all of the framework and jdbc jars in the war, which
is what we want.
+    lcfCrawlerUI.setParentLoaderPriority(true);
+    server.addHandler(lcfCrawlerUI);
+    WebAppContext lcfAuthorityService = new WebAppContext("../../framework/dist/web/war/lcf-authority-service.war","/lcf-authority-service");
+    // This will cause jetty to ignore all of the framework and jdbc jars in the war, which
is what we want.
+    lcfAuthorityService.setParentLoaderPriority(true);
+    server.addHandler(lcfAuthorityService);
+    WebAppContext lcfApi = new WebAppContext("../../framework/dist/web/war/lcf-api.war","/lcf-api");
+    lcfApi.setParentLoaderPriority(true);
+    server.addHandler(lcfApi);
+    server.start();
+
     // If all worked, then we can start the daemon.
     // Clear the agents shutdown signal.
     IThreadContext tc = ThreadContextFactory.make();
@@ -239,6 +314,14 @@ public class TestBase extends org.apache
         if (e != null)
           currentException = e;
       }
+      
+      if (server != null)
+      {
+        server.stop();
+        server.join();
+        server = null;
+      }
+      
       // Clean up everything else
       try
       {
@@ -313,5 +396,5 @@ public class TestBase extends org.apache
     }
     
   }
-  
+
 }

Modified: incubator/lcf/trunk/modules/framework/api/org/apache/lcf/api/APIServlet.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/api/org/apache/lcf/api/APIServlet.java?rev=964422&r1=964421&r2=964422&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/api/org/apache/lcf/api/APIServlet.java (original)
+++ incubator/lcf/trunk/modules/framework/api/org/apache/lcf/api/APIServlet.java Thu Jul 15
13:38:10 2010
@@ -129,7 +129,7 @@ public class APIServlet extends HttpServ
         Configuration input;
         if (argument != null)
         {
-          input = new Argument();
+          input = new Configuration();
           input.fromJSON(argument);
         }
         else

Modified: incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/Configuration.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/Configuration.java?rev=964422&r1=964421&r2=964422&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/Configuration.java
(original)
+++ incubator/lcf/trunk/modules/framework/core/org/apache/lcf/core/interfaces/Configuration.java
Thu Jul 15 13:38:10 2010
@@ -27,7 +27,7 @@ import org.json.*;
 
 /** This class represents XML configuration information, in its most basic incarnation.
 */
-public abstract class Configuration
+public class Configuration
 {
   public static final String _rcsid = "@(#)$Id$";
 
@@ -69,12 +69,18 @@ public abstract class Configuration
   /** Return the root node type.
   *@return the node type name.
   */
-  protected abstract String getRootNodeLabel();
+  protected String getRootNodeLabel()
+  {
+    return "data";
+  }
   
   /** Create a new object of the appropriate class.
   *@return the newly-created configuration object.
   */
-  protected abstract Configuration createNew();
+  protected Configuration createNew()
+  {
+    return new Configuration();
+  }
   
   /** Create a new child node of the appropriate type and class.
   *@return the newly-created node.

Modified: incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/system/LCF.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/system/LCF.java?rev=964422&r1=964421&r2=964422&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/system/LCF.java
(original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/org/apache/lcf/crawler/system/LCF.java
Thu Jul 15 13:38:10 2010
@@ -1006,7 +1006,7 @@ public class LCF extends org.apache.lcf.
   public static Configuration executeCommand(IThreadContext tc, String command, Configuration
inputArgument)
     throws LCFException
   {
-    Configuration rval = new ResponseValue();
+    Configuration rval = new Configuration();
     if (command.equals("job/list"))
     {
       try
@@ -2853,31 +2853,5 @@ public class LCF extends org.apache.lcf.
     return node.getValue();
   }
   
-  protected static class ResponseValue extends Configuration
-  {
-    /** Constructor.
-    */
-    public ResponseValue()
-    {
-      super();
-    }
-
-    /** Return the root node type.
-    *@return the node type name.
-    */
-    protected String getRootNodeLabel()
-    {
-      return "response";
-    }
-    
-    /** Create a new object of the appropriate class.
-    */
-    protected Configuration createNew()
-    {
-      return new ResponseValue();
-    }
-
-  }
-  
 }
 



Mime
View raw message