manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1132374 - in /incubator/lcf/trunk: ./ connectors/filesystem/ framework/ tests/filesystem/org/apache/manifoldcf/filesystem_tests/
Date Sun, 05 Jun 2011 11:58:19 GMT
Author: kwright
Date: Sun Jun  5 11:58:19 2011
New Revision: 1132374

URL: http://svn.apache.org/viewvc?rev=1132374&view=rev
Log:
Finish CONNECTORS-204

Added:
    incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java   (with props)
    incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java   (with props)
    incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java   (with props)
    incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java   (with props)
    incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java   (with props)
Modified:
    incubator/lcf/trunk/   (props changed)
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/build.xml
    incubator/lcf/trunk/connectors/filesystem/   (props changed)
    incubator/lcf/trunk/framework/   (props changed)

Propchange: incubator/lcf/trunk/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Sun Jun  5 11:58:19 2011
@@ -13,6 +13,7 @@ build
 dist
 test-output
 test-postgresql-output
+test-HSQLDB-output
 apache-manifoldcf-0.1-incubator.zip
 apache-manifoldcf-0.1-incubator.tar.gz
 apache-manifoldcf-SNAPSHOT.zip

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1132374&r1=1132373&r2=1132374&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Sun Jun  5 11:58:19 2011
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 0.3-dev =========================
 
+CONNECTORS-204: Add an HSQLDB set of test targets, which use that
+database.
+(Karl Wright)
+
 CONNECTORS-206: Document HSQLDB properties, and mention it as a
 supported choice in the documentation.
 (Karl Wright)

Modified: incubator/lcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/build.xml?rev=1132374&r1=1132373&r2=1132374&view=diff
==============================================================================
--- incubator/lcf/trunk/build.xml (original)
+++ incubator/lcf/trunk/build.xml Sun Jun  5 11:58:19 2011
@@ -1678,6 +1678,41 @@
         </junit>
     </target>
 
+    <target name="run-filesystem-tests-HSQLDB" depends="compile-filesystem-tests,calculate-filesystem-tests-condition" if="filesystem-tests.include">
+        <mkdir dir="test-HSQLDB-output/filesystem"/>
+        <junit fork="true" maxmemory="128m" dir="test-HSQLDB-output/filesystem" showoutput="true" haltonfailure="true">
+            <classpath>
+                <fileset dir="framework/lib">
+                    <include name="*.jar"/>
+                </fileset>
+	      <fileset dir="framework/build/jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="framework/build/test-jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/filesystem/build/jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/filesystem/build/test-jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/nulloutput/build/jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/nulloutput/build/test-jar">
+		<include name="*.jar"/>
+	      </fileset>
+                <pathelement location="build/filesystem-tests/classes"/>
+            </classpath>
+            <test name="org.apache.manifoldcf.filesystem_tests.SanityHSQLDB" todir="test-HSQLDB-output/filesystem"/>
+            <test name="org.apache.manifoldcf.filesystem_tests.APISanityHSQLDB" todir="test-HSQLDB-output/filesystem"/>
+            <test name="org.apache.manifoldcf.filesystem_tests.HopcountHSQLDB" todir="test-HSQLDB-output/filesystem"/>
+            <test name="org.apache.manifoldcf.filesystem_tests.ExpirationHSQLDB" todir="test-HSQLDB-output/filesystem"/>
+            <!-- MHL -->
+        </junit>
+    </target>
+
     <target name="run-sharepoint-tests-postgresql" depends="compile-sharepoint-tests,calculate-sharepoint-tests-condition" if="sharepoint-tests.include">
         <mkdir dir="test-postgresql-output/sharepoint"/>
         <junit fork="true" maxmemory="128m" dir="test-postgresql-output/sharepoint" showoutput="true" haltonfailure="true">
@@ -1709,6 +1744,37 @@
         </junit>
     </target>
 
+    <target name="run-sharepoint-tests-HSQLDB" depends="compile-sharepoint-tests,calculate-sharepoint-tests-condition" if="sharepoint-tests.include">
+        <mkdir dir="test-postgresql-output/sharepoint"/>
+        <junit fork="true" maxmemory="128m" dir="test-HSQLDB-output/sharepoint" showoutput="true" haltonfailure="true">
+            <classpath>
+                <fileset dir="framework/lib">
+                    <include name="*.jar"/>
+                </fileset>
+	      <fileset dir="framework/build/jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="framework/build/test-jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/sharepoint/build/jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/sharepoint/build/test-jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/nulloutput/build/jar">
+		<include name="*.jar"/>
+	      </fileset>
+	      <fileset dir="connectors/nulloutput/build/test-jar">
+		<include name="*.jar"/>
+	      </fileset>
+                <pathelement location="build/sharepoint-tests/classes"/>
+            </classpath>
+            <!-- MHL -->
+        </junit>
+    </target>
+
     <target name="run-tests-open-connectors" depends="run-tests-activedirectory-connector,run-tests-filesystem-connector,run-tests-nullauthority-connector,run-tests-nulloutput-connector,run-tests-rss-connector,run-tests-solr-connector,run-tests-webcrawler-connector,run-tests-jdbc-connector"/>
     <target name="run-tests-lgpl-connectors" depends="run-tests-jcifs-connector"/>
     <target name="run-tests-proprietary-connectors" depends="run-tests-documentum-connector,run-tests-filenet-connector,run-tests-livelink-connector,run-tests-memex-connector,run-tests-meridio-connector,run-tests-sharepoint-connector"/>
@@ -1721,6 +1787,12 @@
     
     <target name="end-to-end-tests-postgresql" depends="run-filesystem-tests-postgresql,run-sharepoint-tests-postgresql"/>
 
+    <target name="run-tests-HSQLDB-open-connectors" depends="run-tests-HSQLDB-activedirectory-connector,run-tests-HSQLDB-filesystem-connector,run-tests-HSQLDB-nullauthority-connector,run-tests-HSQLDB-nulloutput-connector,run-tests-HSQLDB-rss-connector,run-tests-HSQLDB-solr-connector,run-tests-HSQLDB-webcrawler-connector,run-tests-HSQLDB-jdbc-connector"/>
+    <target name="run-tests-HSQLDB-lgpl-connectors" depends="run-tests-HSQLDB-jcifs-connector"/>
+    <target name="run-tests-HSQLDB-proprietary-connectors" depends="run-tests-HSQLDB-documentum-connector,run-tests-HSQLDB-filenet-connector,run-tests-HSQLDB-livelink-connector,run-tests-HSQLDB-memex-connector,run-tests-HSQLDB-meridio-connector,run-tests-HSQLDB-sharepoint-connector"/>
+    
+    <target name="end-to-end-tests-HSQLDB" depends="run-filesystem-tests-HSQLDB,run-sharepoint-tests-HSQLDB"/>
+
     <target name="deliver-open-connectors" depends="deliver-nullauthority-connector,deliver-activedirectory-connector,deliver-filesystem-connector,deliver-rss-connector,deliver-webcrawler-connector,deliver-jdbc-connector"/>
     <target name="deliver-open-connectors-doc" depends="deliver-nullauthority-connector-doc,deliver-activedirectory-connector-doc,deliver-filesystem-connector-doc,deliver-rss-connector-doc,deliver-webcrawler-connector-doc,deliver-jdbc-connector-doc"/>
     
@@ -1808,11 +1880,15 @@
 
     <target name="image" depends="create-source-zip,create-source-tar,create-bin-zip,create-bin-tar"/>
     
-    <target name="test" depends="run-tests-framework,run-tests-open-connectors,run-tests-lgpl-connectors,run-tests-proprietary-connectors,end-to-end-tests"/>
+    <target name="test-dr" depends="run-tests-framework,run-tests-open-connectors,run-tests-lgpl-connectors,run-tests-proprietary-connectors,end-to-end-tests"/>
 
     <target name="test-pg" depends="run-tests-postgresql-framework,run-tests-postgresql-open-connectors,run-tests-postgresql-lgpl-connectors,run-tests-postgresql-proprietary-connectors,end-to-end-tests-postgresql"/>
 
-    <target name="all" depends="build,javadoc,doc,image,test,test-pg"/>
+    <target name="test-hs" depends="run-tests-HSQLDB-framework,run-tests-HSQLDB-open-connectors,run-tests-HSQLDB-lgpl-connectors,run-tests-HSQLDB-proprietary-connectors,end-to-end-tests-HSQLDB"/>
+
+    <target name="test" depends="test-dr,test-hs"/>
+    
+    <target name="all" depends="build,javadoc,doc,image,test-dr,test-pg,test-hs"/>
 
     <target name="help">
       <echo level="info">
@@ -1825,8 +1901,10 @@ clean   - Clean the build area
 build   - Build executable code, jars, processes, and example,
           under the 'dist' directory
 javadoc - Create the javadocs, under the 'dist/doc' directory
-test    - Compile and run tests for all connectors and framework
+test-dr - Compile and run Derby-based tests
 test-pg - Compile and run postgresql-based tests (needs Postgresql)
+test-hs - Compile and run HSQLDB-based tests
+test    - Compile and run all local tests (Derby and HSQLDB)
 doc     - Build html site, under the 'dist/doc' directory
           (requires Apache Forrest)
 image   - Build release image, under the '.' directory (requires

Propchange: incubator/lcf/trunk/connectors/filesystem/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Sun Jun  5 11:58:19 2011
@@ -4,4 +4,6 @@ doc
 lib
 test-output
 test-output-postgresql
+test-output-HSQLDB
+
 

Propchange: incubator/lcf/trunk/framework/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Sun Jun  5 11:58:19 2011
@@ -4,3 +4,5 @@ doc
 lib
 test-output
 test-output-postgresql
+test-output-HSQLDB
+

Added: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java?rev=1132374&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java (added)
+++ incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java Sun Jun  5 11:58:19 2011
@@ -0,0 +1,404 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class APISanityHSQLDB extends TestBaseHSQLDB
+{
+  
+  @Before
+  public void createTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+      createDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void removeTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @Test
+  public void sanityCheck()
+    throws Exception
+  {
+    try
+    {
+      // Hey, we were able to install the file system connector etc.
+      // Now, create a local test job and run it.
+      IThreadContext tc = ThreadContextFactory.make();
+      int i;
+      IJobManager jobManager = JobManagerFactory.make(tc);
+
+      // Create a basic file system connection, and save it.
+      ConfigurationNode connectionObject;
+      ConfigurationNode child;
+      Configuration requestObject;
+      Configuration result;
+      
+      connectionObject = new ConfigurationNode("repositoryconnection");
+      
+      child = new ConfigurationNode("name");
+      child.setValue("File Connection");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+      
+      child = new ConfigurationNode("class_name");
+      child.setValue("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+      
+      child = new ConfigurationNode("description");
+      child.setValue("File Connection");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+
+      child = new ConfigurationNode("max_connections");
+      child.setValue("100");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+
+      requestObject = new Configuration();
+      requestObject.addChild(0,connectionObject);
+      
+      result = performAPIPutOperationViaNodes("repositoryconnections/File%20Connection",201,requestObject);
+      
+      i = 0;
+      while (i < result.getChildCount())
+      {
+        ConfigurationNode resultNode = result.findChild(i++);
+        if (resultNode.getType().equals("error"))
+          throw new Exception(resultNode.getValue());
+      }
+      
+      // Create a basic null output connection, and save it.
+      connectionObject = new ConfigurationNode("outputconnection");
+      
+      child = new ConfigurationNode("name");
+      child.setValue("Null Connection");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+      
+      child = new ConfigurationNode("class_name");
+      child.setValue("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+      
+      child = new ConfigurationNode("description");
+      child.setValue("Null Connection");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+
+      child = new ConfigurationNode("max_connections");
+      child.setValue("100");
+      connectionObject.addChild(connectionObject.getChildCount(),child);
+
+      requestObject = new Configuration();
+      requestObject.addChild(0,connectionObject);
+      
+      result = performAPIPutOperationViaNodes("outputconnections/Null%20Connection",201,requestObject);
+      
+      i = 0;
+      while (i < result.getChildCount())
+      {
+        ConfigurationNode resultNode = result.findChild(i++);
+        if (resultNode.getType().equals("error"))
+          throw new Exception(resultNode.getValue());
+      }
+
+      // Create a job.
+      ConfigurationNode jobObject = new ConfigurationNode("job");
+      
+      child = new ConfigurationNode("description");
+      child.setValue("Test Job");
+      jobObject.addChild(jobObject.getChildCount(),child);
+
+      child = new ConfigurationNode("repository_connection");
+      child.setValue("File Connection");
+      jobObject.addChild(jobObject.getChildCount(),child);
+
+      child = new ConfigurationNode("output_connection");
+      child.setValue("Null Connection");
+      jobObject.addChild(jobObject.getChildCount(),child);
+
+      child = new ConfigurationNode("run_mode");
+      child.setValue("scan once");
+      jobObject.addChild(jobObject.getChildCount(),child);
+
+      child = new ConfigurationNode("start_mode");
+      child.setValue("manual");
+      jobObject.addChild(jobObject.getChildCount(),child);
+
+      child = new ConfigurationNode("hopcount_mode");
+      child.setValue("accurate");
+      jobObject.addChild(jobObject.getChildCount(),child);
+
+      child = new ConfigurationNode("document_specification");
+      // Crawl everything underneath the 'testdata' area
+      File testDataFile = new File("testdata").getCanonicalFile();
+      if (!testDataFile.exists())
+        throw new ManifoldCFException("Test data area not found!  Looking in "+testDataFile.toString());
+      if (!testDataFile.isDirectory())
+        throw new ManifoldCFException("Test data area not a directory!  Looking in "+testDataFile.toString());
+      ConfigurationNode sn = new ConfigurationNode("startpoint");
+      sn.setAttribute("path",testDataFile.toString());
+      ConfigurationNode n = new ConfigurationNode("include");
+      n.setAttribute("type","file");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      n = new ConfigurationNode("include");
+      n.setAttribute("type","directory");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      child.addChild(child.getChildCount(),sn);
+      jobObject.addChild(jobObject.getChildCount(),child);
+      
+      requestObject = new Configuration();
+      requestObject.addChild(0,jobObject);
+      
+      result = performAPIPostOperationViaNodes("jobs",201,requestObject);
+      
+      String jobIDString = null;
+      i = 0;
+      while (i < result.getChildCount())
+      {
+        ConfigurationNode resultNode = result.findChild(i++);
+        if (resultNode.getType().equals("error"))
+          throw new Exception(resultNode.getValue());
+        else if (resultNode.getType().equals("job_id"))
+          jobIDString = resultNode.getValue();
+      }
+      if (jobIDString == null)
+        throw new Exception("Missing job_id from return!");
+      
+      Long jobID = new Long(jobIDString);
+      
+      // Create the test data files.
+      createFile(new File("testdata/test1.txt"),"This is a test file");
+      createFile(new File("testdata/test2.txt"),"This is another test file");
+      createDirectory(new File("testdata/testdir"));
+      createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+      
+      ConfigurationNode requestNode;
+      
+      // Now, start the job, and wait until it completes.
+      startJob(jobIDString);
+      waitJobInactive(jobIDString, 120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      // The test data area has 3 documents and one directory, and we have to count the root directory too.
+      long count;
+      count = getJobDocumentsProcessed(jobIDString);
+      if (count != 5)
+        throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(count).toString());
+      
+      // Add a file and recrawl
+      createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+      // Now, start the job, and wait until it completes.
+      startJob(jobIDString);
+      waitJobInactive(jobIDString, 120000L);
+
+      // The test data area has 4 documents and one directory, and we have to count the root directory too.
+      count = getJobDocumentsProcessed(jobIDString);
+      if (count != 6)
+        throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(count).toString());
+
+      // Change a file, and recrawl
+      changeFile(new File("testdata/test1.txt"),"Modified contents");
+      
+      // Now, start the job, and wait until it completes.
+      startJob(jobIDString);
+      waitJobInactive(jobIDString, 120000L);
+
+      // The test data area has 4 documents and one directory, and we have to count the root directory too.
+      count = getJobDocumentsProcessed(jobIDString);
+      if (count != 6)
+        throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(count).toString());
+      // We also need to make sure the new document was indexed.  Have to think about how to do this though.
+      // MHL
+      
+      // Delete a file, and recrawl
+      removeFile(new File("testdata/test2.txt"));
+      
+      // Now, start the job, and wait until it completes.
+      startJob(jobIDString);
+      waitJobInactive(jobIDString, 120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      // The test data area has 3 documents and one directory, and we have to count the root directory too.
+      count = getJobDocumentsProcessed(jobIDString);
+      if (count != 5)
+        throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(count).toString());
+
+      // Now, delete the job.
+      deleteJob(jobIDString);
+
+      waitJobDeleted(jobIDString, 120000L);
+      
+      // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  protected void startJob(String jobIDString)
+    throws Exception
+  {
+    Configuration requestObject = new Configuration();
+    
+    Configuration result = performAPIPutOperationViaNodes("start/"+jobIDString,201,requestObject);
+    int i = 0;
+    while (i < result.getChildCount())
+    {
+      ConfigurationNode resultNode = result.findChild(i++);
+      if (resultNode.getType().equals("error"))
+        throw new Exception(resultNode.getValue());
+    }
+  }
+  
+  protected void deleteJob(String jobIDString)
+    throws Exception
+  {
+    Configuration result = performAPIDeleteOperationViaNodes("jobs/"+jobIDString,200);
+    int i = 0;
+    while (i < result.getChildCount())
+    {
+      ConfigurationNode resultNode = result.findChild(i++);
+      if (resultNode.getType().equals("error"))
+        throw new Exception(resultNode.getValue());
+    }
+
+  }
+  
+  protected String getJobStatus(String jobIDString)
+    throws Exception
+  {
+    Configuration result = performAPIGetOperationViaNodes("jobstatuses/"+jobIDString,200);
+    String status = null;
+    int i = 0;
+    while (i < result.getChildCount())
+    {
+      ConfigurationNode resultNode = result.findChild(i++);
+      if (resultNode.getType().equals("error"))
+        throw new Exception(resultNode.getValue());
+      else if (resultNode.getType().equals("jobstatus"))
+      {
+        int j = 0;
+        while (j < resultNode.getChildCount())
+        {
+          ConfigurationNode childNode = resultNode.findChild(j++);
+          if (childNode.getType().equals("status"))
+            status = childNode.getValue();
+        }
+      }
+    }
+    return status;
+  }
+
+  protected long getJobDocumentsProcessed(String jobIDString)
+    throws Exception
+  {
+    Configuration result = performAPIGetOperationViaNodes("jobstatuses/"+jobIDString,200);
+    String documentsProcessed = null;
+    int i = 0;
+    while (i < result.getChildCount())
+    {
+      ConfigurationNode resultNode = result.findChild(i++);
+      if (resultNode.getType().equals("error"))
+        throw new Exception(resultNode.getValue());
+      else if (resultNode.getType().equals("jobstatus"))
+      {
+        int j = 0;
+        while (j < resultNode.getChildCount())
+        {
+          ConfigurationNode childNode = resultNode.findChild(j++);
+          if (childNode.getType().equals("documents_processed"))
+            documentsProcessed = childNode.getValue();
+        }
+      }
+    }
+    if (documentsProcessed == null)
+      throw new Exception("Expected a documents_processed field, didn't find it");
+    return new Long(documentsProcessed).longValue();
+  }
+
+  protected void waitJobInactive(String jobIDString, long maxTime)
+    throws Exception
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      String status = getJobStatus(jobIDString);
+      if (status == null)
+        throw new Exception("No such job: '"+jobIDString+"'");
+      if (status.equals("not yet run"))
+        throw new Exception("Job was never started.");
+      if (status.equals("done"))
+        return;
+      if (status.equals("error"))
+        throw new Exception("Job reports error.");
+      ManifoldCF.sleep(1000L);
+      continue;
+    }
+    throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+  
+  protected void waitJobDeleted(String jobIDString, long maxTime)
+    throws Exception
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      String status = getJobStatus(jobIDString);
+      if (status == null)
+        return;
+      ManifoldCF.sleep(1000L);
+    }
+    throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+    
+
+}

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/APISanityHSQLDB.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java?rev=1132374&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java (added)
+++ incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java Sun Jun  5 11:58:19 2011
@@ -0,0 +1,259 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to see if document expiration works properly. */
+public class ExpirationHSQLDB extends TestBaseHSQLDB
+{
+  
+  @Before
+  public void createTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+      createDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void removeTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @Test
+  public void expirationCheck()
+    throws Exception
+  {
+    try
+    {
+      // Hey, we were able to install the file system connector etc.
+      // Now, create a local test job and run it.
+      IThreadContext tc = ThreadContextFactory.make();
+      
+      // Create a basic file system connection, and save it.
+      IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+      IRepositoryConnection conn = mgr.create();
+      conn.setName("File Connection");
+      conn.setDescription("File Connection");
+      conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+      conn.setMaxConnections(100);
+      // Now, save
+      mgr.save(conn);
+      
+      // Create a basic null output connection, and save it.
+      IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+      IOutputConnection outputConn = outputMgr.create();
+      outputConn.setName("Null Connection");
+      outputConn.setDescription("Null Connection");
+      outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+      outputConn.setMaxConnections(100);
+      // Now, save
+      outputMgr.save(outputConn);
+
+      // Create a job.
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      IJobDescription job = jobManager.createJob();
+      job.setDescription("Test Job");
+      job.setConnectionName("File Connection");
+      job.setOutputConnectionName("Null Connection");
+      job.setType(job.TYPE_SPECIFIED);
+      job.setStartMethod(job.START_DISABLE);
+      job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      // 60-second expiration for all documents..
+      job.setExpiration(new Long(60000L));
+      // Infinite rescan interval.
+      job.setInterval(null);
+      // Continuous job.
+      job.setType(IJobDescription.TYPE_CONTINUOUS);
+      
+      // Now, set up the document specification.
+      DocumentSpecification ds = job.getSpecification();
+      // Crawl everything underneath the 'testdata' area
+      File testDataFile = new File("testdata").getCanonicalFile();
+      if (!testDataFile.exists())
+        throw new ManifoldCFException("Test data area not found!  Looking in "+testDataFile.toString());
+      if (!testDataFile.isDirectory())
+        throw new ManifoldCFException("Test data area not a directory!  Looking in "+testDataFile.toString());
+      SpecificationNode sn = new SpecificationNode("startpoint");
+      sn.setAttribute("path",testDataFile.toString());
+      SpecificationNode n = new SpecificationNode("include");
+      n.setAttribute("type","file");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      n = new SpecificationNode("include");
+      n.setAttribute("type","directory");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      ds.addChild(ds.getChildCount(),sn);
+      
+      // Set up the output specification.
+      OutputSpecification os = job.getOutputSpecification();
+      // Null output connections have no output specification, so this is a no-op.
+      
+      // Save the job.
+      jobManager.save(job);
+
+      // Create the test data files.
+      createFile(new File("testdata/test1.txt"),"This is a test file");
+      createFile(new File("testdata/test2.txt"),"This is another test file");
+      createDirectory(new File("testdata/testdir"));
+      createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+      createDirectory(new File("testdata/testdir/seconddir"));
+      createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
+      
+      // Now, start the job, and wait until it is running.
+      jobManager.manualStart(job.getID());
+      waitJobRunning(jobManager,job.getID(),30000L);
+      
+      JobStatus status;
+      
+      // Now we wait, and we should see 7 documents eventually.
+      long startTime = System.currentTimeMillis();
+      while (System.currentTimeMillis() < startTime + 120000L)
+      {
+        status = jobManager.getStatus(job.getID());
+        if (status.getDocumentsProcessed() == 7)
+          break;
+
+        ManifoldCF.sleep(1000L);
+      }
+      
+      // At this point there should be 7 documents.
+      // OK, documents should expire starting a minute later.  The number of documents will go quickly to zero after this time. 
+      // So all we need to do is confirm that the job stops within 2 minutes.
+      waitJobInactive(jobManager,job.getID(),180000L);
+
+      status = jobManager.getStatus(job.getID());
+      if (status.getDocumentsProcessed() != 0)
+        throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString());
+      
+
+      // Now, delete the job.
+      jobManager.deleteJob(job.getID());
+      waitJobDeleted(jobManager,job.getID(),120000L);
+      
+      // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        throw new ManifoldCFException("No such job: '"+jobID+"'");
+      int statusValue = status.getStatus();
+      switch (statusValue)
+      {
+        case JobStatus.JOBSTATUS_NOTYETRUN:
+          throw new ManifoldCFException("Job was never started.");
+        case JobStatus.JOBSTATUS_COMPLETED:
+          break;
+        case JobStatus.JOBSTATUS_ERROR:
+          throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+        default:
+          ManifoldCF.sleep(1000L);
+          continue;
+      }
+      return;
+    }
+    throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+  
+  protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        throw new ManifoldCFException("No such job: '"+jobID+"'");
+      int statusValue = status.getStatus();
+      switch (statusValue)
+      {
+        case JobStatus.JOBSTATUS_NOTYETRUN:
+          throw new ManifoldCFException("Job was never started.");
+        case JobStatus.JOBSTATUS_COMPLETED:
+          throw new ManifoldCFException("Job ended on its own!");
+        case JobStatus.JOBSTATUS_ERROR:
+          throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+        case JobStatus.JOBSTATUS_RUNNING:
+          break;
+        default:
+          ManifoldCF.sleep(1000L);
+          continue;
+      }
+      return;
+    }
+    throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+
+  protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        return;
+      ManifoldCF.sleep(1000L);
+    }
+    throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+    
+
+}

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDB.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java?rev=1132374&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java (added)
+++ incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java Sun Jun  5 11:58:19 2011
@@ -0,0 +1,209 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test which checks to be sure hopcount functionality is working properly. */
+public class HopcountHSQLDB extends TestBaseHSQLDB
+{
+  
+  @Before
+  public void createTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+      createDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void removeTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @Test
+  public void hopcountCheck()
+    throws Exception
+  {
+    try
+    {
+      // Hey, we were able to install the file system connector etc.
+      // Now, create a local test job and run it.
+      IThreadContext tc = ThreadContextFactory.make();
+      
+      // Create a basic file system connection, and save it.
+      IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+      IRepositoryConnection conn = mgr.create();
+      conn.setName("File Connection");
+      conn.setDescription("File Connection");
+      conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+      conn.setMaxConnections(100);
+      // Now, save
+      mgr.save(conn);
+      
+      // Create a basic null output connection, and save it.
+      IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+      IOutputConnection outputConn = outputMgr.create();
+      outputConn.setName("Null Connection");
+      outputConn.setDescription("Null Connection");
+      outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+      outputConn.setMaxConnections(100);
+      // Now, save
+      outputMgr.save(outputConn);
+
+      // Create a job.
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      IJobDescription job = jobManager.createJob();
+      job.setDescription("Test Job");
+      job.setConnectionName("File Connection");
+      job.setOutputConnectionName("Null Connection");
+      job.setType(job.TYPE_SPECIFIED);
+      job.setStartMethod(job.START_DISABLE);
+      job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      job.addHopCountFilter("child",new Long(2));
+      
+      // Now, set up the document specification.
+      DocumentSpecification ds = job.getSpecification();
+      // Crawl everything underneath the 'testdata' area
+      File testDataFile = new File("testdata").getCanonicalFile();
+      if (!testDataFile.exists())
+        throw new ManifoldCFException("Test data area not found!  Looking in "+testDataFile.toString());
+      if (!testDataFile.isDirectory())
+        throw new ManifoldCFException("Test data area not a directory!  Looking in "+testDataFile.toString());
+      SpecificationNode sn = new SpecificationNode("startpoint");
+      sn.setAttribute("path",testDataFile.toString());
+      SpecificationNode n = new SpecificationNode("include");
+      n.setAttribute("type","file");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      n = new SpecificationNode("include");
+      n.setAttribute("type","directory");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      ds.addChild(ds.getChildCount(),sn);
+      
+      // Set up the output specification.
+      OutputSpecification os = job.getOutputSpecification();
+      // Null output connections have no output specification, so this is a no-op.
+      
+      // Save the job.
+      jobManager.save(job);
+
+      // Create the test data files.
+      createFile(new File("testdata/test1.txt"),"This is a test file");
+      createFile(new File("testdata/test2.txt"),"This is another test file");
+      createDirectory(new File("testdata/testdir"));
+      createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+      createDirectory(new File("testdata/testdir/seconddir"));
+      createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file");
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      JobStatus status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and 2 directories and we have to count the root directory too.
+      // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7.
+      if (status.getDocumentsProcessed() != 6)
+        throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+      
+      // Now, delete the job.
+      jobManager.deleteJob(job.getID());
+      waitJobDeleted(jobManager,job.getID(), 120000L);
+      
+      // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        throw new ManifoldCFException("No such job: '"+jobID+"'");
+      int statusValue = status.getStatus();
+      switch (statusValue)
+      {
+        case JobStatus.JOBSTATUS_NOTYETRUN:
+          throw new ManifoldCFException("Job was never started.");
+        case JobStatus.JOBSTATUS_COMPLETED:
+          break;
+        case JobStatus.JOBSTATUS_ERROR:
+          throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+        default:
+          ManifoldCF.sleep(1000L);
+          continue;
+      }
+      return;
+    }
+    throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+  
+  protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        return;
+      ManifoldCF.sleep(1000L);
+    }
+    throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+    
+
+}

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDB.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java?rev=1132374&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java (added)
+++ incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java Sun Jun  5 11:58:19 2011
@@ -0,0 +1,244 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a very basic sanity check */
+public class SanityHSQLDB extends TestBaseHSQLDB
+{
+  
+  @Before
+  public void createTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+      createDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void removeTestArea()
+    throws Exception
+  {
+    try
+    {
+      File f = new File("testdata");
+      removeDirectory(f);
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @Test
+  public void sanityCheck()
+    throws Exception
+  {
+    try
+    {
+      // Hey, we were able to install the file system connector etc.
+      // Now, create a local test job and run it.
+      IThreadContext tc = ThreadContextFactory.make();
+      
+      // Create a basic file system connection, and save it.
+      IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+      IRepositoryConnection conn = mgr.create();
+      conn.setName("File Connection");
+      conn.setDescription("File Connection");
+      conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+      conn.setMaxConnections(100);
+      // Now, save
+      mgr.save(conn);
+      
+      // Create a basic null output connection, and save it.
+      IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+      IOutputConnection outputConn = outputMgr.create();
+      outputConn.setName("Null Connection");
+      outputConn.setDescription("Null Connection");
+      outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector");
+      outputConn.setMaxConnections(100);
+      // Now, save
+      outputMgr.save(outputConn);
+
+      // Create a job.
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      IJobDescription job = jobManager.createJob();
+      job.setDescription("Test Job");
+      job.setConnectionName("File Connection");
+      job.setOutputConnectionName("Null Connection");
+      job.setType(job.TYPE_SPECIFIED);
+      job.setStartMethod(job.START_DISABLE);
+      job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      
+      // Now, set up the document specification.
+      DocumentSpecification ds = job.getSpecification();
+      // Crawl everything underneath the 'testdata' area
+      File testDataFile = new File("testdata").getCanonicalFile();
+      if (!testDataFile.exists())
+        throw new ManifoldCFException("Test data area not found!  Looking in "+testDataFile.toString());
+      if (!testDataFile.isDirectory())
+        throw new ManifoldCFException("Test data area not a directory!  Looking in "+testDataFile.toString());
+      SpecificationNode sn = new SpecificationNode("startpoint");
+      sn.setAttribute("path",testDataFile.toString());
+      SpecificationNode n = new SpecificationNode("include");
+      n.setAttribute("type","file");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      n = new SpecificationNode("include");
+      n.setAttribute("type","directory");
+      n.setAttribute("match","*");
+      sn.addChild(sn.getChildCount(),n);
+      ds.addChild(ds.getChildCount(),sn);
+      
+      // Set up the output specification.
+      OutputSpecification os = job.getOutputSpecification();
+      // Null output connections have no output specification, so this is a no-op.
+      
+      // Save the job.
+      jobManager.save(job);
+
+      // Create the test data files.
+      createFile(new File("testdata/test1.txt"),"This is a test file");
+      createFile(new File("testdata/test2.txt"),"This is another test file");
+      createDirectory(new File("testdata/testdir"));
+      createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file");
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      JobStatus status = jobManager.getStatus(job.getID());
+      // The test data area has 3 documents and one directory, and we have to count the root directory too.
+      if (status.getDocumentsProcessed() != 5)
+        throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
+      
+      // Add a file and recrawl
+      createFile(new File("testdata/testdir/test4.txt"),"Added file");
+
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and one directory, and we have to count the root directory too.
+      if (status.getDocumentsProcessed() != 6)
+        throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+      // Change a file, and recrawl
+      changeFile(new File("testdata/test1.txt"),"Modified contents");
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 4 documents and one directory, and we have to count the root directory too.
+      if (status.getDocumentsProcessed() != 6)
+        throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+      // We also need to make sure the new document was indexed.  Have to think about how to do this though.
+      // MHL
+      
+      // Delete a file, and recrawl
+      removeFile(new File("testdata/test2.txt"));
+      
+      // Now, start the job, and wait until it completes.
+      jobManager.manualStart(job.getID());
+      waitJobInactive(jobManager,job.getID(),120000L);
+
+      // Check to be sure we actually processed the right number of documents.
+      status = jobManager.getStatus(job.getID());
+      // The test data area has 3 documents and one directory, and we have to count the root directory too.
+      if (status.getDocumentsProcessed() != 5)
+        throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+      // Now, delete the job.
+      jobManager.deleteJob(job.getID());
+      waitJobDeleted(jobManager,job.getID(),120000L);
+      
+      // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around.
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        throw new ManifoldCFException("No such job: '"+jobID+"'");
+      int statusValue = status.getStatus();
+      switch (statusValue)
+      {
+        case JobStatus.JOBSTATUS_NOTYETRUN:
+          throw new ManifoldCFException("Job was never started.");
+        case JobStatus.JOBSTATUS_COMPLETED:
+          break;
+        case JobStatus.JOBSTATUS_ERROR:
+          throw new ManifoldCFException("Job reports error status: "+status.getErrorText());
+        default:
+          ManifoldCF.sleep(1000L);
+          continue;
+      }
+      return;
+    }
+    throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+  
+  protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime)
+    throws ManifoldCFException, InterruptedException
+  {
+    long startTime = System.currentTimeMillis();
+    while (System.currentTimeMillis() < startTime + maxTime)
+    {
+      JobStatus status = jobManager.getStatus(jobID);
+      if (status == null)
+        return;
+      ManifoldCF.sleep(1000L);
+    }
+    throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds");
+  }
+    
+
+}

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/SanityHSQLDB.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java?rev=1132374&view=auto
==============================================================================
--- incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java (added)
+++ incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java Sun Jun  5 11:58:19 2011
@@ -0,0 +1,511 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.filesystem_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.mortbay.log.Logger;
+
+import org.apache.commons.httpclient.*;
+import org.apache.commons.httpclient.methods.*;
+
+/** Tests that run the "agents daemon" should be derived from this */
+public class TestBaseHSQLDB extends org.apache.manifoldcf.crawler.tests.TestConnectorBaseHSQLDB
+{
+  public static final String agentShutdownSignal = "agent-process";
+  public static final int testPort = 8346;
+  
+  protected DaemonThread daemonThread = null;
+  protected Server server = null;
+
+  protected String[] getConnectorNames()
+  {
+    return new String[]{"File Connector"};
+  }
+  
+  protected String[] getConnectorClasses()
+  {
+    return new String[]{"org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"};
+  }
+  
+  protected String[] getOutputNames()
+  {
+    return new String[]{"Null Output"};
+  }
+  
+  protected String[] getOutputClasses()
+  {
+    return new String[]{"org.apache.manifoldcf.agents.output.nullconnector.NullConnector"};
+  }
+  
+  protected void createDirectory(File f)
+    throws Exception
+  {
+    if (f.mkdirs() == false)
+      throw new Exception("Failed to create directory "+f.toString());
+  }
+  
+  protected void removeDirectory(File f)
+    throws Exception
+  {
+    File[] files = f.listFiles();
+    if (files != null)
+    {
+      int i = 0;
+      while (i < files.length)
+      {
+        File subfile = files[i++];
+        if (subfile.isDirectory())
+          removeDirectory(subfile);
+        else
+          subfile.delete();
+      }
+    }
+    f.delete();
+  }
+  
+  protected void createFile(File f, String contents)
+    throws Exception
+  {
+    OutputStream os = new FileOutputStream(f);
+    try
+    {
+      Writer w = new OutputStreamWriter(os,"utf-8");
+      try
+      {
+        w.write(contents);
+      }
+      finally
+      {
+        w.flush();
+      }
+    }
+    finally
+    {
+      os.close();
+    }
+  }
+  
+  protected void removeFile(File f)
+    throws Exception
+  {
+    if (f.delete() == false)
+      throw new Exception("Failed to delete file "+f.toString());
+  }
+  
+  protected void changeFile(File f, String newContents)
+    throws Exception
+  {
+    removeFile(f);
+    createFile(f,newContents);
+  }
+  
+  // API support
+  
+  // These methods allow communication with the ManifoldCF api webapp, via the locally-instantiated jetty
+  
+  /** Construct a command url.
+  */
+  protected String makeAPIURL(String command)
+  {
+    return "http://localhost:"+Integer.toString(testPort)+"/mcf-api-service/json/"+command;
+  }
+  
+  /** Perform an json API GET operation.
+  *@param apiURL is the operation.
+  *@param expectedResponse is the expected response code.
+  *@return the json response.
+  */
+  protected String performAPIGetOperation(String apiURL, int expectedResponse)
+    throws Exception
+  {
+    HttpClient client = new HttpClient();
+    GetMethod method = new GetMethod(apiURL);
+    int response = client.executeMethod(method);
+    byte[] responseData = method.getResponseBody();
+    String responseString = new String(responseData,"utf-8");
+    if (response != expectedResponse)
+      throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+    // We presume that the data is utf-8, since that's what the API uses throughout.
+    return responseString;
+  }
+
+  /** Perform an json API DELETE operation.
+  *@param apiURL is the operation.
+  *@param expectedResponse is the expected response code.
+  *@return the json response.
+  */
+  protected String performAPIDeleteOperation(String apiURL, int expectedResponse)
+    throws Exception
+  {
+    HttpClient client = new HttpClient();
+    DeleteMethod method = new DeleteMethod(apiURL);
+    int response = client.executeMethod(method);
+    byte[] responseData = method.getResponseBody();
+    String responseString = new String(responseData,"utf-8");
+    if (response != expectedResponse)
+      throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+    // We presume that the data is utf-8, since that's what the API uses throughout.
+    return responseString;
+  }
+
+  /** Perform an json API PUT operation.
+  *@param apiURL is the operation.
+  *@param input is the input JSON.
+  *@param expectedResponse is the expected response code.
+  *@return the json response.
+  */
+  protected String performAPIPutOperation(String apiURL, int expectedResponse, String input)
+    throws Exception
+  {
+    HttpClient client = new HttpClient();
+    PutMethod method = new PutMethod(apiURL);
+    method.setRequestHeader("Content-type", "text/plain; charset=UTF-8");
+    method.setRequestBody(input);
+    int response = client.executeMethod(method);
+    byte[] responseData = method.getResponseBody();
+    String responseString = new String(responseData,"utf-8");
+    if (response != expectedResponse)
+      throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+    // We presume that the data is utf-8, since that's what the API uses throughout.
+    return responseString;
+  }
+
+  /** Perform an json API POST operation.
+  *@param apiURL is the operation.
+  *@param input is the input JSON.
+  *@param expectedResponse is the expected response code.
+  *@return the json response.
+  */
+  protected String performAPIPostOperation(String apiURL, int expectedResponse, String input)
+    throws Exception
+  {
+    HttpClient client = new HttpClient();
+    PostMethod method = new PostMethod(apiURL);
+    method.setRequestHeader("Content-type", "text/plain; charset=UTF-8");
+    method.setRequestBody(input);
+    int response = client.executeMethod(method);
+    byte[] responseData = method.getResponseBody();
+    String responseString = new String(responseData,"utf-8");
+    if (response != expectedResponse)
+      throw new Exception("API http error; expected "+Integer.toString(expectedResponse)+", saw "+Integer.toString(response)+": "+responseString);
+    // We presume that the data is utf-8, since that's what the API uses throughout.
+    return responseString;
+  }
+
+  /** Perform a json GET API operation, using Configuration structures to represent the json.  This is for testing convenience,
+  * mostly.
+  */
+  protected Configuration performAPIGetOperationViaNodes(String command, int expectedResponse)
+    throws Exception
+  {
+    String result = performAPIGetOperation(makeAPIURL(command),expectedResponse);
+    Configuration cfg = new Configuration();
+    cfg.fromJSON(result);
+    return cfg;
+  }
+
+  /** Perform a json DELETE API operation, using Configuration structures to represent the json.  This is for testing convenience,
+  * mostly.
+  */
+  protected Configuration performAPIDeleteOperationViaNodes(String command, int expectedResponse)
+    throws Exception
+  {
+    String result = performAPIDeleteOperation(makeAPIURL(command),expectedResponse);
+    Configuration cfg = new Configuration();
+    cfg.fromJSON(result);
+    return cfg;
+  }
+
+  /** Perform a json PUT API operation, using Configuration structures to represent the json.  This is for testing convenience,
+  * mostly.
+  */
+  protected Configuration performAPIPutOperationViaNodes(String command, int expectedResponse, Configuration argument)
+    throws Exception
+  {
+    String argumentJson;
+    if (argument != null)
+      argumentJson = argument.toJSON();
+    else
+      argumentJson = null;
+    
+    String result = performAPIPutOperation(makeAPIURL(command),expectedResponse,argumentJson);
+    Configuration cfg = new Configuration();
+    cfg.fromJSON(result);
+    return cfg;
+  }
+
+  /** Perform a json POST API operation, using Configuration structures to represent the json.  This is for testing convenience,
+  * mostly.
+  */
+  protected Configuration performAPIPostOperationViaNodes(String command, int expectedResponse, Configuration argument)
+    throws Exception
+  {
+    String argumentJson;
+    if (argument != null)
+      argumentJson = argument.toJSON();
+    else
+      argumentJson = null;
+    
+    String result = performAPIPostOperation(makeAPIURL(command),expectedResponse,argumentJson);
+    Configuration cfg = new Configuration();
+    cfg.fromJSON(result);
+    return cfg;
+  }
+
+  // Setup/teardown
+  
+  @Before
+  public void setUp()
+    throws Exception
+  {
+    super.setUp();
+    // Start jetty
+    server = new Server( testPort );    
+    server.setStopAtShutdown( true );
+    
+    // Initialize the servlets
+    WebAppContext lcfCrawlerUI = new WebAppContext("../../framework/dist/web/war/mcf-crawler-ui.war","/mcf-crawler-ui");
+    // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+    lcfCrawlerUI.setParentLoaderPriority(true);
+    server.addHandler(lcfCrawlerUI);
+    WebAppContext lcfAuthorityService = new WebAppContext("../../framework/dist/web/war/mcf-authority-service.war","/mcf-authority-service");
+    // This will cause jetty to ignore all of the framework and jdbc jars in the war, which is what we want.
+    lcfAuthorityService.setParentLoaderPriority(true);
+    server.addHandler(lcfAuthorityService);
+    WebAppContext lcfApi = new WebAppContext("../../framework/dist/web/war/mcf-api-service.war","/mcf-api-service");
+    lcfApi.setParentLoaderPriority(true);
+    server.addHandler(lcfApi);
+    server.start();
+
+    // If all worked, then we can start the daemon.
+    // Clear the agents shutdown signal.
+    IThreadContext tc = ThreadContextFactory.make();
+    ILockManager lockManager = LockManagerFactory.make(tc);
+    lockManager.clearGlobalFlag(agentShutdownSignal);
+
+    daemonThread = new DaemonThread();
+    daemonThread.start();
+  }
+  
+  @After
+  public void cleanUp()
+    throws Exception
+  {
+    initialize();
+    if (isInitialized())
+    {
+      Exception currentException = null;
+      IThreadContext tc = ThreadContextFactory.make();
+
+      // Delete all jobs (and wait for them to go away)
+      if (daemonThread != null)
+      {
+        IJobManager jobManager = JobManagerFactory.make(tc);
+        
+        // Get a list of the current active jobs
+        IJobDescription[] jobs = jobManager.getAllJobs();
+        int i = 0;
+        while (i < jobs.length)
+        {
+          IJobDescription desc = jobs[i++];
+          // Abort this job, if it is running
+          try
+          {
+            jobManager.manualAbort(desc.getID());
+          }
+          catch (ManifoldCFException e)
+          {
+            // This generally means that the job was not running
+          }
+        }
+        i = 0;
+        while (i < jobs.length)
+        {
+          IJobDescription desc = jobs[i++];
+          // Wait for this job to stop
+          while (true)
+          {
+            JobStatus status = jobManager.getStatus(desc.getID());
+            if (status != null)
+            {
+              int statusValue = status.getStatus();
+              switch (statusValue)
+              {
+              case JobStatus.JOBSTATUS_NOTYETRUN:
+              case JobStatus.JOBSTATUS_COMPLETED:
+              case JobStatus.JOBSTATUS_ERROR:
+                break;
+              default:
+                ManifoldCF.sleep(10000);
+                continue;
+              }
+            }
+            break;
+          }
+        }
+
+        // Now, delete them all
+        i = 0;
+        while (i < jobs.length)
+        {
+          IJobDescription desc = jobs[i++];
+          try
+          {
+            jobManager.deleteJob(desc.getID());
+          }
+          catch (ManifoldCFException e)
+          {
+            // This usually means that the job is already being deleted
+          }
+        }
+
+        i = 0;
+        while (i < jobs.length)
+        {
+          IJobDescription desc = jobs[i++];
+          // Wait for this job to disappear
+          while (true)
+          {
+            JobStatus status = jobManager.getStatus(desc.getID());
+            if (status != null)
+            {
+              ManifoldCF.sleep(10000);
+              continue;
+            }
+            break;
+          }
+        }
+
+        // Shut down daemon
+        ILockManager lockManager = LockManagerFactory.make(tc);
+        lockManager.setGlobalFlag(agentShutdownSignal);
+      
+        // Wait for daemon thread to exit.
+        while (true)
+        {
+          if (daemonThread.isAlive())
+          {
+            Thread.sleep(1000L);
+            continue;
+          }
+          break;
+        }
+
+        Exception e = daemonThread.getDaemonException();
+        if (e != null)
+          currentException = e;
+      }
+      
+      if (server != null)
+      {
+        server.stop();
+        server.join();
+        server = null;
+      }
+      
+      // Clean up everything else
+      try
+      {
+        super.cleanUp();
+      }
+      catch (Exception e)
+      {
+        if (currentException == null)
+          currentException = e;
+      }
+      if (currentException != null)
+        throw currentException;
+    }
+  }
+  
+  protected static class DaemonThread extends Thread
+  {
+    protected Exception daemonException = null;
+    
+    public DaemonThread()
+    {
+      setName("Daemon thread");
+    }
+    
+    public void run()
+    {
+      IThreadContext tc = ThreadContextFactory.make();
+      // Now, start the server, and then wait for the shutdown signal.  On shutdown, we have to actually do the cleanup,
+      // because the JVM isn't going away.
+      try
+      {
+        ILockManager lockManager = LockManagerFactory.make(tc);
+        while (true)
+        {
+          // Any shutdown signal yet?
+          if (lockManager.checkGlobalFlag(agentShutdownSignal))
+            break;
+            
+          // Start whatever agents need to be started
+          ManifoldCF.startAgents(tc);
+
+          try
+          {
+            ManifoldCF.sleep(5000);
+          }
+          catch (InterruptedException e)
+          {
+            break;
+          }
+        }
+      }
+      catch (ManifoldCFException e)
+      {
+        daemonException = e;
+      }
+      finally
+      {
+        try
+        {
+          ManifoldCF.stopAgents(tc);
+        }
+        catch (ManifoldCFException e)
+        {
+          daemonException = e;
+        }
+      }
+    }
+    
+    public Exception getDaemonException()
+    {
+      return daemonException;
+    }
+    
+  }
+
+}

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/tests/filesystem/org/apache/manifoldcf/filesystem_tests/TestBaseHSQLDB.java
------------------------------------------------------------------------------
    svn:keywords = Id



Mime
View raw message