Return-Path: X-Original-To: apmail-incubator-connectors-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-connectors-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 43BDFB5C1 for ; Sat, 31 Dec 2011 17:24:05 +0000 (UTC) Received: (qmail 11820 invoked by uid 500); 31 Dec 2011 17:24:04 -0000 Delivered-To: apmail-incubator-connectors-commits-archive@incubator.apache.org Received: (qmail 11685 invoked by uid 500); 31 Dec 2011 17:24:04 -0000 Mailing-List: contact connectors-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: connectors-dev@incubator.apache.org Delivered-To: mailing list connectors-commits@incubator.apache.org Received: (qmail 11676 invoked by uid 99); 31 Dec 2011 17:24:04 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 Dec 2011 17:24:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 Dec 2011 17:24:01 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A2B7723889D7; Sat, 31 Dec 2011 17:23:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1226141 [3/5] - in /incubator/lcf/trunk: ./ connectors/alfresco/ connectors/alfresco/alfresco-ws-client/ connectors/cmis/ connectors/documentum/dfc/ connectors/filenet/filenet-api/ connectors/filesystem/ connectors/jcifs/jcifs/ connectors/... Date: Sat, 31 Dec 2011 17:23:37 -0000 To: connectors-commits@incubator.apache.org From: kwright@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111231172340.A2B7723889D7@eris.apache.org> Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlMySQLLT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,211 +25,31 @@ import org.junit.*; /** This is a very basic sanity check */ public class BigCrawlMySQLLT extends BaseMySQL { + protected BigCrawlTester tester; + + public BigCrawlMySQLLT() + { + tester = new BigCrawlTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - // Create the test data files. - String baseFileName = "testdata/"; - int i0 = 0; - while (i0 < 10) - { - String fileName0 = baseFileName + "/dir-" + i0; - createDirectory(new File(fileName0)); - int i1 = 0; - while (i1 < 10) - { - String fileName1 = fileName0 + "/dir-" + i1; - createDirectory(new File(fileName1)); - int i2 = 0; - while (i2 < 10) - { - String fileName2 = fileName1 + "/dir-" + i2; - createDirectory(new File(fileName2)); - int i3 = 0; - while (i3 < 10) - { - String fileName3 = fileName2 + "/dir-" + i3; - createDirectory(new File(fileName3)); - int i4 = 0; - while (i4 < 10) - { - String fileName4 = fileName3 + "/file-"+i4; - createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4); - i4++; - } - i3++; - } - i2++; - } - i1++; - } - i0++; - } - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - System.out.println("Done generating files"); + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - System.out.println("Removing generated files"); - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void bigCrawl() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_NEVERDELETE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Now, start the job, and wait until it completes. - long startTime = System.currentTimeMillis(); - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),18000000L); - System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds"); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 111111) - throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),18000000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlPostgresqlLT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,211 +25,31 @@ import org.junit.*; /** This is a very basic sanity check */ public class BigCrawlPostgresqlLT extends BasePostgresql { + protected BigCrawlTester tester; + + public BigCrawlPostgresqlLT() + { + tester = new BigCrawlTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - // Create the test data files. - String baseFileName = "testdata/"; - int i0 = 0; - while (i0 < 10) - { - String fileName0 = baseFileName + "/dir-" + i0; - createDirectory(new File(fileName0)); - int i1 = 0; - while (i1 < 10) - { - String fileName1 = fileName0 + "/dir-" + i1; - createDirectory(new File(fileName1)); - int i2 = 0; - while (i2 < 10) - { - String fileName2 = fileName1 + "/dir-" + i2; - createDirectory(new File(fileName2)); - int i3 = 0; - while (i3 < 10) - { - String fileName3 = fileName2 + "/dir-" + i3; - createDirectory(new File(fileName3)); - int i4 = 0; - while (i4 < 10) - { - String fileName4 = fileName3 + "/file-"+i4; - createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4); - i4++; - } - i3++; - } - i2++; - } - i1++; - } - i0++; - } - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - System.out.println("Done generating files"); + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - System.out.println("Removing generated files"); - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void bigCrawl() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_NEVERDELETE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Now, start the job, and wait until it completes. - long startTime = System.currentTimeMillis(); - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),18000000L); - System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds"); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 111111) - throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),18000000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java?rev=1226141&view=auto ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java (added) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java Sat Dec 31 17:23:35 2011 @@ -0,0 +1,176 @@ +/* $Id$ */ + +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.manifoldcf.filesystem_tests; + +import org.apache.manifoldcf.core.interfaces.*; +import org.apache.manifoldcf.agents.interfaces.*; +import org.apache.manifoldcf.crawler.interfaces.*; +import org.apache.manifoldcf.crawler.system.ManifoldCF; + +import java.io.*; +import java.util.*; + +/** This is a 100000 document crawl */ +public class BigCrawlTester +{ + protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance; + + public BigCrawlTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance) + { + this.instance = instance; + } + + public void setupTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + FileHelper.createDirectory(f); + // Create the test data files. + String baseFileName = "testdata/"; + int i0 = 0; + while (i0 < 10) + { + String fileName0 = baseFileName + "/dir-" + i0; + FileHelper.createDirectory(new File(fileName0)); + int i1 = 0; + while (i1 < 10) + { + String fileName1 = fileName0 + "/dir-" + i1; + FileHelper.createDirectory(new File(fileName1)); + int i2 = 0; + while (i2 < 10) + { + String fileName2 = fileName1 + "/dir-" + i2; + FileHelper.createDirectory(new File(fileName2)); + int i3 = 0; + while (i3 < 10) + { + String fileName3 = fileName2 + "/dir-" + i3; + FileHelper.createDirectory(new File(fileName3)); + int i4 = 0; + while (i4 < 10) + { + String fileName4 = fileName3 + "/file-"+i4; + FileHelper.createFile(new File(fileName4),"Test file "+i0+":"+i1+":"+i2+":"+i3+":"+i4); + i4++; + } + i3++; + } + i2++; + } + i1++; + } + i0++; + } + System.err.println("Done generating files"); + } + + public void teardownTestArea() + throws Exception + { + System.err.println("Removing generated files"); + File f = new File("testdata"); + FileHelper.removeDirectory(f); + } + + public void executeTest() + throws Exception + { + // Hey, we were able to install the file system connector etc. + // Now, create a local test job and run it. + IThreadContext tc = ThreadContextFactory.make(); + + // Create a basic file system connection, and save it. + IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); + IRepositoryConnection conn = mgr.create(); + conn.setName("File Connection"); + conn.setDescription("File Connection"); + conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); + conn.setMaxConnections(100); + // Now, save + mgr.save(conn); + + // Create a basic null output connection, and save it. + IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); + IOutputConnection outputConn = outputMgr.create(); + outputConn.setName("Null Connection"); + outputConn.setDescription("Null Connection"); + outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); + outputConn.setMaxConnections(100); + // Now, save + outputMgr.save(outputConn); + + // Create a job. + IJobManager jobManager = JobManagerFactory.make(tc); + IJobDescription job = jobManager.createJob(); + job.setDescription("Test Job"); + job.setConnectionName("File Connection"); + job.setOutputConnectionName("Null Connection"); + job.setType(job.TYPE_SPECIFIED); + job.setStartMethod(job.START_DISABLE); + job.setHopcountMode(job.HOPCOUNT_NEVERDELETE); + + // Now, set up the document specification. + DocumentSpecification ds = job.getSpecification(); + // Crawl everything underneath the 'testdata' area + File testDataFile = new File("testdata").getCanonicalFile(); + if (!testDataFile.exists()) + throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); + if (!testDataFile.isDirectory()) + throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); + SpecificationNode sn = new SpecificationNode("startpoint"); + sn.setAttribute("path",testDataFile.toString()); + SpecificationNode n = new SpecificationNode("include"); + n.setAttribute("type","file"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + n = new SpecificationNode("include"); + n.setAttribute("type","directory"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + ds.addChild(ds.getChildCount(),sn); + + // Set up the output specification. + OutputSpecification os = job.getOutputSpecification(); + // Null output connections have no output specification, so this is a no-op. + + // Save the job. + jobManager.save(job); + + // Now, start the job, and wait until it completes. + long startTime = System.currentTimeMillis(); + jobManager.manualStart(job.getID()); + instance.waitJobInactiveNative(jobManager,job.getID(),18000000L); + System.err.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds"); + + // Check to be sure we actually processed the right number of documents. + JobStatus status = jobManager.getStatus(job.getID()); + // The test data area has 3 documents and one directory, and we have to count the root directory too. + if (status.getDocumentsProcessed() != 111111) + throw new ManifoldCFException("Wrong number of documents processed - expected 111111, saw "+new Long(status.getDocumentsProcessed()).toString()); + + // Now, delete the job. + jobManager.deleteJob(job.getID()); + instance.waitJobDeletedNative(jobManager,job.getID(),18000000L); + + // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. + } + +} Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/BigCrawlTester.java ------------------------------------------------------------------------------ svn:keywords = Id Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationHSQLDBIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,230 +25,32 @@ import org.junit.*; /** This is a test which checks to see if document expiration works properly. */ public class ExpirationHSQLDBIT extends BaseHSQLDB { + protected ExpirationTester tester; + + public ExpirationHSQLDBIT() + { + tester = new ExpirationTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void expirationCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - // 60-second expiration for all documents.. - job.setExpiration(new Long(60000L)); - // Infinite rescan interval. - job.setInterval(null); - // Continuous job. - job.setType(IJobDescription.TYPE_CONTINUOUS); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it is running. - jobManager.manualStart(job.getID()); - waitJobRunning(jobManager,job.getID(),30000L); - - JobStatus status; - - // Now we wait, and we should see 7 documents eventually. - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + 120000L) - { - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() == 7) - break; - - ManifoldCF.sleep(1000L); - } - - // At this point there should be 7 documents. - // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time. - // So all we need to do is confirm that the job stops within 2 minutes. - waitJobInactive(jobManager,job.getID(),180000L); - - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() != 0) - throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString()); - - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - throw new ManifoldCFException("Job ended on its own!"); - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - case JobStatus.JOBSTATUS_RUNNING: - break; - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,230 +25,31 @@ import org.junit.*; /** This is a test which checks to see if document expiration works properly. */ public class ExpirationIT extends BaseDerby { + protected ExpirationTester tester; + + public ExpirationIT() + { + tester = new ExpirationTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void expirationCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - // 60-second expiration for all documents.. - job.setExpiration(new Long(60000L)); - // Infinite rescan interval. - job.setInterval(null); - // Continuous job. - job.setType(IJobDescription.TYPE_CONTINUOUS); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it is running. - jobManager.manualStart(job.getID()); - waitJobRunning(jobManager,job.getID(),30000L); - - JobStatus status; - - // Now we wait, and we should see 7 documents eventually. - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + 120000L) - { - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() == 7) - break; - - ManifoldCF.sleep(1000L); - } - - // At this point there should be 7 documents. - // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time. - // So all we need to do is confirm that the job stops within 2 minutes. - waitJobInactive(jobManager,job.getID(),180000L); - - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() != 0) - throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString()); - - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.executeTest(); } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - throw new ManifoldCFException("Job ended on its own!"); - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - case JobStatus.JOBSTATUS_RUNNING: - break; - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationMySQLIT.java Sat Dec 31 17:23:35 2011 @@ -30,230 +30,31 @@ import org.junit.*; /** This is a test which checks to see if document expiration works properly. */ public class ExpirationMySQLIT extends BaseMySQL { + protected ExpirationTester tester; + + public ExpirationMySQLIT() + { + tester = new ExpirationTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void expirationCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - // 60-second expiration for all documents.. - job.setExpiration(new Long(60000L)); - // Infinite rescan interval. - job.setInterval(null); - // Continuous job. - job.setType(IJobDescription.TYPE_CONTINUOUS); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it is running. - jobManager.manualStart(job.getID()); - waitJobRunning(jobManager,job.getID(),30000L); - - JobStatus status; - - // Now we wait, and we should see 7 documents eventually. - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + 120000L) - { - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() == 7) - break; - - ManifoldCF.sleep(1000L); - } - - // At this point there should be 7 documents. - // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time. - // So all we need to do is confirm that the job stops within 2 minutes. - waitJobInactive(jobManager,job.getID(),180000L); - - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() != 0) - throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString()); - - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - throw new ManifoldCFException("Job ended on its own!"); - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - case JobStatus.JOBSTATUS_RUNNING: - break; - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationPostgresqlIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,230 +25,31 @@ import org.junit.*; /** This is a test which checks to see if document expiration works properly. */ public class ExpirationPostgresqlIT extends BasePostgresql { + protected ExpirationTester tester; + + public ExpirationPostgresqlIT() + { + tester = new ExpirationTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void expirationCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - // 60-second expiration for all documents.. - job.setExpiration(new Long(60000L)); - // Infinite rescan interval. - job.setInterval(null); - // Continuous job. - job.setType(IJobDescription.TYPE_CONTINUOUS); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it is running. - jobManager.manualStart(job.getID()); - waitJobRunning(jobManager,job.getID(),30000L); - - JobStatus status; - - // Now we wait, and we should see 7 documents eventually. - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + 120000L) - { - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() == 7) - break; - - ManifoldCF.sleep(1000L); - } - - // At this point there should be 7 documents. - // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time. - // So all we need to do is confirm that the job stops within 2 minutes. - waitJobInactive(jobManager,job.getID(),180000L); - - status = jobManager.getStatus(job.getID()); - if (status.getDocumentsProcessed() != 0) - throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString()); - - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.executeTest(); } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobRunning(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - throw new ManifoldCFException("Job ended on its own!"); - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - case JobStatus.JOBSTATUS_RUNNING: - break; - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not start in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java?rev=1226141&view=auto ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java (added) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java Sat Dec 31 17:23:35 2011 @@ -0,0 +1,166 @@ +/* $Id$ */ + +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.manifoldcf.filesystem_tests; + +import org.apache.manifoldcf.core.interfaces.*; +import org.apache.manifoldcf.agents.interfaces.*; +import org.apache.manifoldcf.crawler.interfaces.*; +import org.apache.manifoldcf.crawler.system.ManifoldCF; + +import java.io.*; +import java.util.*; + +/** This is a test which checks to see if document expiration works properly. */ +public class ExpirationTester +{ + protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance; + + public ExpirationTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance) + { + this.instance = instance; + } + + public void setupTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + FileHelper.createDirectory(f); + } + + public void teardownTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + } + + public void executeTest() + throws Exception + { + // Hey, we were able to install the file system connector etc. + // Now, create a local test job and run it. + IThreadContext tc = ThreadContextFactory.make(); + + // Create a basic file system connection, and save it. + IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); + IRepositoryConnection conn = mgr.create(); + conn.setName("File Connection"); + conn.setDescription("File Connection"); + conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); + conn.setMaxConnections(100); + // Now, save + mgr.save(conn); + + // Create a basic null output connection, and save it. + IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); + IOutputConnection outputConn = outputMgr.create(); + outputConn.setName("Null Connection"); + outputConn.setDescription("Null Connection"); + outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); + outputConn.setMaxConnections(100); + // Now, save + outputMgr.save(outputConn); + + // Create a job. + IJobManager jobManager = JobManagerFactory.make(tc); + IJobDescription job = jobManager.createJob(); + job.setDescription("Test Job"); + job.setConnectionName("File Connection"); + job.setOutputConnectionName("Null Connection"); + job.setType(job.TYPE_SPECIFIED); + job.setStartMethod(job.START_DISABLE); + job.setHopcountMode(job.HOPCOUNT_ACCURATE); + // 60-second expiration for all documents.. + job.setExpiration(new Long(60000L)); + // Infinite rescan interval. + job.setInterval(null); + // Continuous job. + job.setType(IJobDescription.TYPE_CONTINUOUS); + + // Now, set up the document specification. + DocumentSpecification ds = job.getSpecification(); + // Crawl everything underneath the 'testdata' area + File testDataFile = new File("testdata").getCanonicalFile(); + if (!testDataFile.exists()) + throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); + if (!testDataFile.isDirectory()) + throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); + SpecificationNode sn = new SpecificationNode("startpoint"); + sn.setAttribute("path",testDataFile.toString()); + SpecificationNode n = new SpecificationNode("include"); + n.setAttribute("type","file"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + n = new SpecificationNode("include"); + n.setAttribute("type","directory"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + ds.addChild(ds.getChildCount(),sn); + + // Set up the output specification. + OutputSpecification os = job.getOutputSpecification(); + // Null output connections have no output specification, so this is a no-op. + + // Save the job. + jobManager.save(job); + + // Create the test data files. + FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file"); + FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file"); + FileHelper.createDirectory(new File("testdata/testdir")); + FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); + FileHelper.createDirectory(new File("testdata/testdir/seconddir")); + FileHelper.createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); + + // Now, start the job, and wait until it is running. + jobManager.manualStart(job.getID()); + instance.waitJobRunningNative(jobManager,job.getID(),30000L); + + JobStatus status; + + // Now we wait, and we should see 7 documents eventually. + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() < startTime + 120000L) + { + status = jobManager.getStatus(job.getID()); + if (status.getDocumentsProcessed() == 7) + break; + + ManifoldCF.sleep(1000L); + } + + // At this point there should be 7 documents. + // OK, documents should expire starting a minute later. The number of documents will go quickly to zero after this time. + // So all we need to do is confirm that the job stops within 2 minutes. + instance.waitJobInactiveNative(jobManager,job.getID(),180000L); + + status = jobManager.getStatus(job.getID()); + if (status.getDocumentsProcessed() != 0) + throw new ManifoldCFException("Wrong number of documents processed - expected 0, saw "+new Long(status.getDocumentsProcessed()).toString()); + + + // Now, delete the job. + jobManager.deleteJob(job.getID()); + instance.waitJobDeletedNative(jobManager,job.getID(),120000L); + + // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. + } + +} Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/ExpirationTester.java ------------------------------------------------------------------------------ svn:keywords = Id Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java?rev=1226141&view=auto ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java (added) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java Sat Dec 31 17:23:35 2011 @@ -0,0 +1,90 @@ +/* $Id$ */ + +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.manifoldcf.filesystem_tests; + +import java.io.*; +import java.util.*; + +/** Helper class for local file manipulation */ +public class FileHelper +{ + + protected static void createDirectory(File f) + throws Exception + { + if (f.mkdirs() == false) + throw new Exception("Failed to create directory "+f.toString()); + } + + protected static void removeDirectory(File f) + throws Exception + { + File[] files = f.listFiles(); + if (files != null) + { + int i = 0; + while (i < files.length) + { + File subfile = files[i++]; + if (subfile.isDirectory()) + removeDirectory(subfile); + else + subfile.delete(); + } + } + f.delete(); + } + + protected static void createFile(File f, String contents) + throws Exception + { + OutputStream os = new FileOutputStream(f); + try + { + Writer w = new OutputStreamWriter(os,"utf-8"); + try + { + w.write(contents); + } + finally + { + w.flush(); + } + } + finally + { + os.close(); + } + } + + protected static void removeFile(File f) + throws Exception + { + if (f.delete() == false) + throw new Exception("Failed to delete file "+f.toString()); + } + + protected static void changeFile(File f, String newContents) + throws Exception + { + removeFile(f); + createFile(f,newContents); + } + +} Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/FileHelper.java ------------------------------------------------------------------------------ svn:keywords = Id Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountHSQLDBIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,180 +25,32 @@ import org.junit.*; /** This is a test which checks to be sure hopcount functionality is working properly. */ public class HopcountHSQLDBIT extends BaseHSQLDB { + protected HopcountTester tester; + + public HopcountHSQLDBIT() + { + tester = new HopcountTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void hopcountCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - job.addHopCountFilter("child",new Long(2)); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and 2 directories and we have to count the root directory too. - // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(), 120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.executeTest(); } - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - }