Return-Path: X-Original-To: apmail-incubator-connectors-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-connectors-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5FD2FB5D6 for ; Sat, 31 Dec 2011 17:24:05 +0000 (UTC) Received: (qmail 11898 invoked by uid 500); 31 Dec 2011 17:24:05 -0000 Delivered-To: apmail-incubator-connectors-commits-archive@incubator.apache.org Received: (qmail 11841 invoked by uid 500); 31 Dec 2011 17:24:04 -0000 Mailing-List: contact connectors-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: connectors-dev@incubator.apache.org Delivered-To: mailing list connectors-commits@incubator.apache.org Received: (qmail 11825 invoked by uid 99); 31 Dec 2011 17:24:04 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 Dec 2011 17:24:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 Dec 2011 17:24:01 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A51DC2388A56; Sat, 31 Dec 2011 17:23:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1226141 [4/5] - in /incubator/lcf/trunk: ./ connectors/alfresco/ connectors/alfresco/alfresco-ws-client/ connectors/cmis/ connectors/documentum/dfc/ connectors/filenet/filenet-api/ connectors/filesystem/ connectors/jcifs/jcifs/ connectors/... Date: Sat, 31 Dec 2011 17:23:37 -0000 To: connectors-commits@incubator.apache.org From: kwright@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111231172340.A51DC2388A56@eris.apache.org> Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,180 +25,32 @@ import org.junit.*; /** This is a test which checks to be sure hopcount functionality is working properly. */ public class HopcountIT extends BaseDerby { + protected HopcountTester tester; + + public HopcountIT() + { + tester = new HopcountTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void hopcountCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - job.addHopCountFilter("child",new Long(2)); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and 2 directories and we have to count the root directory too. - // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(), 120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.executeTest(); } - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountMySQLIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountMySQLIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountMySQLIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountMySQLIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,180 +25,31 @@ import org.junit.*; /** This is a test which checks to be sure hopcount functionality is working properly. */ public class HopcountMySQLIT extends BaseMySQL { + protected HopcountTester tester; + + public HopcountMySQLIT() + { + tester = new HopcountTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void hopcountCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - job.addHopCountFilter("child",new Long(2)); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and 2 directories and we have to count the root directory too. - // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(), 120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountPostgresqlIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountPostgresqlIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountPostgresqlIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountPostgresqlIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,180 +25,31 @@ import org.junit.*; /** This is a test which checks to be sure hopcount functionality is working properly. */ public class HopcountPostgresqlIT extends BasePostgresql { + protected HopcountTester tester; + + public HopcountPostgresqlIT() + { + tester = new HopcountTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void hopcountCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - job.addHopCountFilter("child",new Long(2)); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - createDirectory(new File("testdata/testdir/seconddir")); - createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and 2 directories and we have to count the root directory too. - // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(), 120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - } Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java?rev=1226141&view=auto ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java (added) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java Sat Dec 31 17:23:35 2011 @@ -0,0 +1,145 @@ +/* $Id$ */ + +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.manifoldcf.filesystem_tests; + +import org.apache.manifoldcf.core.interfaces.*; +import org.apache.manifoldcf.agents.interfaces.*; +import org.apache.manifoldcf.crawler.interfaces.*; +import org.apache.manifoldcf.crawler.system.ManifoldCF; + +import java.io.*; +import java.util.*; + +/** This is a test which checks to be sure hopcount functionality is working properly. */ +public class HopcountTester +{ + protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance; + + public HopcountTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance) + { + this.instance = instance; + } + + public void setupTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + FileHelper.createDirectory(f); + } + + public void teardownTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + } + + public void executeTest() + throws Exception + { + // Hey, we were able to install the file system connector etc. + // Now, create a local test job and run it. + IThreadContext tc = ThreadContextFactory.make(); + + // Create a basic file system connection, and save it. + IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); + IRepositoryConnection conn = mgr.create(); + conn.setName("File Connection"); + conn.setDescription("File Connection"); + conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); + conn.setMaxConnections(100); + // Now, save + mgr.save(conn); + + // Create a basic null output connection, and save it. + IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); + IOutputConnection outputConn = outputMgr.create(); + outputConn.setName("Null Connection"); + outputConn.setDescription("Null Connection"); + outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); + outputConn.setMaxConnections(100); + // Now, save + outputMgr.save(outputConn); + + // Create a job. + IJobManager jobManager = JobManagerFactory.make(tc); + IJobDescription job = jobManager.createJob(); + job.setDescription("Test Job"); + job.setConnectionName("File Connection"); + job.setOutputConnectionName("Null Connection"); + job.setType(job.TYPE_SPECIFIED); + job.setStartMethod(job.START_DISABLE); + job.setHopcountMode(job.HOPCOUNT_ACCURATE); + job.addHopCountFilter("child",new Long(2)); + + // Now, set up the document specification. + DocumentSpecification ds = job.getSpecification(); + // Crawl everything underneath the 'testdata' area + File testDataFile = new File("testdata").getCanonicalFile(); + if (!testDataFile.exists()) + throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); + if (!testDataFile.isDirectory()) + throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); + SpecificationNode sn = new SpecificationNode("startpoint"); + sn.setAttribute("path",testDataFile.toString()); + SpecificationNode n = new SpecificationNode("include"); + n.setAttribute("type","file"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + n = new SpecificationNode("include"); + n.setAttribute("type","directory"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + ds.addChild(ds.getChildCount(),sn); + + // Set up the output specification. + OutputSpecification os = job.getOutputSpecification(); + // Null output connections have no output specification, so this is a no-op. + + // Save the job. + jobManager.save(job); + + // Create the test data files. + FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file"); + FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file"); + FileHelper.createDirectory(new File("testdata/testdir")); + FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); + FileHelper.createDirectory(new File("testdata/testdir/seconddir")); + FileHelper.createFile(new File("testdata/testdir/seconddir/test4.txt"),"Lowest test file"); + + // Now, start the job, and wait until it completes. + jobManager.manualStart(job.getID()); + instance.waitJobInactiveNative(jobManager,job.getID(),120000L); + + // Check to be sure we actually processed the right number of documents. + JobStatus status = jobManager.getStatus(job.getID()); + // The test data area has 4 documents and 2 directories and we have to count the root directory too. + // But the max hopcount is 2, so one file will be left behind, so the count should be 6, not 7. + if (status.getDocumentsProcessed() != 6) + throw new ManifoldCFException("Wrong number of documents processed - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); + + // Now, delete the job. + jobManager.deleteJob(job.getID()); + instance.waitJobDeletedNative(jobManager,job.getID(), 120000L); + + // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. + } + +} Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/HopcountTester.java ------------------------------------------------------------------------------ svn:keywords = Id Copied: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationDerbyUI.java (from r1225824, incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationUI.java) URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationDerbyUI.java?p2=incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationDerbyUI.java&p1=incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationUI.java&r1=1225824&r2=1226141&rev=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationUI.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/NavigationDerbyUI.java Sat Dec 31 17:23:35 2011 @@ -30,7 +30,7 @@ import org.junit.*; import org.apache.manifoldcf.core.tests.HTMLTester; /** Basic UI navigation tests */ -public class NavigationUI extends BaseUIDerby +public class NavigationDerbyUI extends BaseUIDerby { @Test Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityHSQLDBIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityHSQLDBIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityHSQLDBIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityHSQLDBIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,215 +25,32 @@ import org.junit.*; /** This is a very basic sanity check */ public class SanityHSQLDBIT extends BaseHSQLDB { + protected SanityTester tester; + + public SanityHSQLDBIT() + { + tester = new SanityTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void sanityCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Add a file and recrawl - createFile(new File("testdata/testdir/test4.txt"),"Added file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Change a file, and recrawl - changeFile(new File("testdata/test1.txt"),"Modified contents"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - // We also need to make sure the new document was indexed. Have to think about how to do this though. - // MHL - - // Delete a file, and recrawl - removeFile(new File("testdata/test2.txt")); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - + } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,215 +25,32 @@ import org.junit.*; /** This is a very basic sanity check */ public class SanityIT extends BaseDerby { + protected SanityTester tester; + + public SanityIT() + { + tester = new SanityTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void sanityCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Add a file and recrawl - createFile(new File("testdata/testdir/test4.txt"),"Added file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Change a file, and recrawl - changeFile(new File("testdata/test1.txt"),"Modified contents"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - // We also need to make sure the new document was indexed. Have to think about how to do this though. - // MHL - - // Delete a file, and recrawl - removeFile(new File("testdata/test2.txt")); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - + } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityMySQLIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityMySQLIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityMySQLIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityMySQLIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,215 +25,32 @@ import org.junit.*; /** This is a very basic sanity check */ public class SanityMySQLIT extends BaseMySQL { + protected SanityTester tester; + + public SanityMySQLIT() + { + tester = new SanityTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void sanityCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Add a file and recrawl - createFile(new File("testdata/testdir/test4.txt"),"Added file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Change a file, and recrawl - changeFile(new File("testdata/test1.txt"),"Modified contents"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - // We also need to make sure the new document was indexed. Have to think about how to do this though. - // MHL - - // Delete a file, and recrawl - removeFile(new File("testdata/test2.txt")); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - + } Modified: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityPostgresqlIT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityPostgresqlIT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityPostgresqlIT.java (original) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityPostgresqlIT.java Sat Dec 31 17:23:35 2011 @@ -18,11 +18,6 @@ */ package org.apache.manifoldcf.filesystem_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - import java.io.*; import java.util.*; import org.junit.*; @@ -30,215 +25,32 @@ import org.junit.*; /** This is a very basic sanity check */ public class SanityPostgresqlIT extends BasePostgresql { + protected SanityTester tester; + + public SanityPostgresqlIT() + { + tester = new SanityTester(mcfInstance); + } @Before - public void createTestArea() + public void setupTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - createDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.setupTestArea(); } @After - public void removeTestArea() + public void teardownTester() throws Exception { - try - { - File f = new File("testdata"); - removeDirectory(f); - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.teardownTestArea(); } @Test public void sanityCheck() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("File Connection"); - conn.setDescription("File Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); - conn.setMaxConnections(100); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("File Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_ACCURATE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // Crawl everything underneath the 'testdata' area - File testDataFile = new File("testdata").getCanonicalFile(); - if (!testDataFile.exists()) - throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); - if (!testDataFile.isDirectory()) - throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); - SpecificationNode sn = new SpecificationNode("startpoint"); - sn.setAttribute("path",testDataFile.toString()); - SpecificationNode n = new SpecificationNode("include"); - n.setAttribute("type","file"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - n = new SpecificationNode("include"); - n.setAttribute("type","directory"); - n.setAttribute("match","*"); - sn.addChild(sn.getChildCount(),n); - ds.addChild(ds.getChildCount(),sn); - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Create the test data files. - createFile(new File("testdata/test1.txt"),"This is a test file"); - createFile(new File("testdata/test2.txt"),"This is another test file"); - createDirectory(new File("testdata/testdir")); - createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Add a file and recrawl - createFile(new File("testdata/testdir/test4.txt"),"Added file"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Change a file, and recrawl - changeFile(new File("testdata/test1.txt"),"Modified contents"); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - status = jobManager.getStatus(job.getID()); - // The test data area has 4 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 6) - throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); - // We also need to make sure the new document was indexed. Have to think about how to do this though. - // MHL - - // Delete a file, and recrawl - removeFile(new File("testdata/test2.txt")); - - // Now, start the job, and wait until it completes. - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),120000L); - - // Check to be sure we actually processed the right number of documents. - status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 5) - throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),120000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } - } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); + tester.executeTest(); } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - + } Added: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java?rev=1226141&view=auto ============================================================================== --- incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java (added) +++ incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java Sat Dec 31 17:23:35 2011 @@ -0,0 +1,180 @@ +/* $Id$ */ + +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.manifoldcf.filesystem_tests; + +import org.apache.manifoldcf.core.interfaces.*; +import org.apache.manifoldcf.agents.interfaces.*; +import org.apache.manifoldcf.crawler.interfaces.*; +import org.apache.manifoldcf.crawler.system.ManifoldCF; + +import java.io.*; +import java.util.*; + +/** This is a very basic sanity check */ +public class SanityTester +{ + protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance; + + public SanityTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance) + { + this.instance = instance; + } + + public void setupTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + FileHelper.createDirectory(f); + } + + public void teardownTestArea() + throws Exception + { + File f = new File("testdata"); + FileHelper.removeDirectory(f); + } + + public void executeTest() + throws Exception + { + // Hey, we were able to install the file system connector etc. + // Now, create a local test job and run it. + IThreadContext tc = ThreadContextFactory.make(); + + // Create a basic file system connection, and save it. + IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); + IRepositoryConnection conn = mgr.create(); + conn.setName("File Connection"); + conn.setDescription("File Connection"); + conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector"); + conn.setMaxConnections(100); + // Now, save + mgr.save(conn); + + // Create a basic null output connection, and save it. + IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); + IOutputConnection outputConn = outputMgr.create(); + outputConn.setName("Null Connection"); + outputConn.setDescription("Null Connection"); + outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); + outputConn.setMaxConnections(100); + // Now, save + outputMgr.save(outputConn); + + // Create a job. + IJobManager jobManager = JobManagerFactory.make(tc); + IJobDescription job = jobManager.createJob(); + job.setDescription("Test Job"); + job.setConnectionName("File Connection"); + job.setOutputConnectionName("Null Connection"); + job.setType(job.TYPE_SPECIFIED); + job.setStartMethod(job.START_DISABLE); + job.setHopcountMode(job.HOPCOUNT_ACCURATE); + + // Now, set up the document specification. + DocumentSpecification ds = job.getSpecification(); + // Crawl everything underneath the 'testdata' area + File testDataFile = new File("testdata").getCanonicalFile(); + if (!testDataFile.exists()) + throw new ManifoldCFException("Test data area not found! Looking in "+testDataFile.toString()); + if (!testDataFile.isDirectory()) + throw new ManifoldCFException("Test data area not a directory! Looking in "+testDataFile.toString()); + SpecificationNode sn = new SpecificationNode("startpoint"); + sn.setAttribute("path",testDataFile.toString()); + SpecificationNode n = new SpecificationNode("include"); + n.setAttribute("type","file"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + n = new SpecificationNode("include"); + n.setAttribute("type","directory"); + n.setAttribute("match","*"); + sn.addChild(sn.getChildCount(),n); + ds.addChild(ds.getChildCount(),sn); + + // Set up the output specification. + OutputSpecification os = job.getOutputSpecification(); + // Null output connections have no output specification, so this is a no-op. + + // Save the job. + jobManager.save(job); + + // Create the test data files. + FileHelper.createFile(new File("testdata/test1.txt"),"This is a test file"); + FileHelper.createFile(new File("testdata/test2.txt"),"This is another test file"); + FileHelper.createDirectory(new File("testdata/testdir")); + FileHelper.createFile(new File("testdata/testdir/test3.txt"),"This is yet another test file"); + + // Now, start the job, and wait until it completes. + jobManager.manualStart(job.getID()); + instance.waitJobInactiveNative(jobManager,job.getID(),120000L); + + // Check to be sure we actually processed the right number of documents. + JobStatus status = jobManager.getStatus(job.getID()); + // The test data area has 3 documents and one directory, and we have to count the root directory too. + if (status.getDocumentsProcessed() != 5) + throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); + + // Add a file and recrawl + FileHelper.createFile(new File("testdata/testdir/test4.txt"),"Added file"); + + // Now, start the job, and wait until it completes. + jobManager.manualStart(job.getID()); + instance.waitJobInactiveNative(jobManager,job.getID(),120000L); + + status = jobManager.getStatus(job.getID()); + // The test data area has 4 documents and one directory, and we have to count the root directory too. + if (status.getDocumentsProcessed() != 6) + throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); + + // Change a file, and recrawl + FileHelper.changeFile(new File("testdata/test1.txt"),"Modified contents"); + + // Now, start the job, and wait until it completes. + jobManager.manualStart(job.getID()); + instance.waitJobInactiveNative(jobManager,job.getID(),120000L); + + status = jobManager.getStatus(job.getID()); + // The test data area has 4 documents and one directory, and we have to count the root directory too. + if (status.getDocumentsProcessed() != 6) + throw new ManifoldCFException("Wrong number of documents processed after change - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString()); + // We also need to make sure the new document was indexed. Have to think about how to do this though. + // MHL + + // Delete a file, and recrawl + FileHelper.removeFile(new File("testdata/test2.txt")); + + // Now, start the job, and wait until it completes. + jobManager.manualStart(job.getID()); + instance.waitJobInactiveNative(jobManager,job.getID(),120000L); + + // Check to be sure we actually processed the right number of documents. + status = jobManager.getStatus(job.getID()); + // The test data area has 3 documents and one directory, and we have to count the root directory too. + if (status.getDocumentsProcessed() != 5) + throw new ManifoldCFException("Wrong number of documents processed after delete - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString()); + + // Now, delete the job. + jobManager.deleteJob(job.getID()); + instance.waitJobDeletedNative(jobManager,job.getID(),120000L); + + // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. + } + +} Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/lcf/trunk/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java ------------------------------------------------------------------------------ svn:keywords = Id Modified: incubator/lcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/BigCrawlDerbyLT.java URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/BigCrawlDerbyLT.java?rev=1226141&r1=1226140&r2=1226141&view=diff ============================================================================== --- incubator/lcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/BigCrawlDerbyLT.java (original) +++ incubator/lcf/trunk/tests/rss/src/test/java/org/apache/manifoldcf/rss_tests/BigCrawlDerbyLT.java Sat Dec 31 17:23:35 2011 @@ -18,13 +18,6 @@ */ package org.apache.manifoldcf.rss_tests; -import org.apache.manifoldcf.core.interfaces.*; -import org.apache.manifoldcf.agents.interfaces.*; -import org.apache.manifoldcf.crawler.interfaces.*; -import org.apache.manifoldcf.crawler.system.ManifoldCF; - -import org.apache.manifoldcf.crawler.connectors.rss.RSSConnector; - import java.io.*; import java.util.*; import org.junit.*; @@ -33,8 +26,14 @@ import org.junit.*; public class BigCrawlDerbyLT extends BaseDerby { + protected BigCrawlTester tester; protected MockRSSService rssService = null; + public BigCrawlDerbyLT() + { + tester = new BigCrawlTester(mcfInstance); + } + // Setup and teardown the mock wiki service @Before @@ -57,130 +56,6 @@ public class BigCrawlDerbyLT extends Bas public void bigCrawl() throws Exception { - try - { - // Hey, we were able to install the file system connector etc. - // Now, create a local test job and run it. - IThreadContext tc = ThreadContextFactory.make(); - - // Create a basic file system connection, and save it. - IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc); - IRepositoryConnection conn = mgr.create(); - conn.setName("RSS Connection"); - conn.setDescription("RSS Connection"); - conn.setClassName("org.apache.manifoldcf.crawler.connectors.rss.RSSConnector"); - conn.setMaxConnections(100); - ConfigParams cp = conn.getConfigParams(); - cp.setParameter(RSSConnector.emailParameter,"somebody@somewhere.com"); - cp.setParameter(RSSConnector.maxOpenParameter,"100"); - cp.setParameter(RSSConnector.maxFetchesParameter,"1000000"); - cp.setParameter(RSSConnector.bandwidthParameter,"1000000"); - cp.setParameter(RSSConnector.robotsUsageParameter,"none"); - // Now, save - mgr.save(conn); - - // Create a basic null output connection, and save it. - IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc); - IOutputConnection outputConn = outputMgr.create(); - outputConn.setName("Null Connection"); - outputConn.setDescription("Null Connection"); - outputConn.setClassName("org.apache.manifoldcf.agents.output.nullconnector.NullConnector"); - outputConn.setMaxConnections(100); - // Now, save - outputMgr.save(outputConn); - - // Create a job. - IJobManager jobManager = JobManagerFactory.make(tc); - IJobDescription job = jobManager.createJob(); - job.setDescription("Test Job"); - job.setConnectionName("RSS Connection"); - job.setOutputConnectionName("Null Connection"); - job.setType(job.TYPE_SPECIFIED); - job.setStartMethod(job.START_DISABLE); - job.setHopcountMode(job.HOPCOUNT_NEVERDELETE); - - // Now, set up the document specification. - DocumentSpecification ds = job.getSpecification(); - // For 100000 documents, set up 10000 seeds - for (int i = 0 ; i < 10000 ; i++) - { - SpecificationNode sn = new SpecificationNode("feed"); - sn.setAttribute("url","http://localhost:8189/rss/gen.php?type=feed&feed="+i); - ds.addChild(ds.getChildCount(),sn); - } - - // Set up the output specification. - OutputSpecification os = job.getOutputSpecification(); - // Null output connections have no output specification, so this is a no-op. - - // Save the job. - jobManager.save(job); - - // Now, start the job, and wait until it completes. - long startTime = System.currentTimeMillis(); - jobManager.manualStart(job.getID()); - waitJobInactive(jobManager,job.getID(),22000000L); - System.out.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+" milliseconds"); - - // Check to be sure we actually processed the right number of documents. - JobStatus status = jobManager.getStatus(job.getID()); - // The test data area has 3 documents and one directory, and we have to count the root directory too. - if (status.getDocumentsProcessed() != 110000) - throw new ManifoldCFException("Wrong number of documents processed - expected 110000, saw "+new Long(status.getDocumentsProcessed()).toString()); - - // Now, delete the job. - jobManager.deleteJob(job.getID()); - waitJobDeleted(jobManager,job.getID(),18000000L); - - // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections lying around. - } - catch (Exception e) - { - e.printStackTrace(); - throw e; - } + tester.executeTest(); } - - protected void waitJobInactive(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - throw new ManifoldCFException("No such job: '"+jobID+"'"); - int statusValue = status.getStatus(); - switch (statusValue) - { - case JobStatus.JOBSTATUS_NOTYETRUN: - throw new ManifoldCFException("Job was never started."); - case JobStatus.JOBSTATUS_COMPLETED: - break; - case JobStatus.JOBSTATUS_ERROR: - throw new ManifoldCFException("Job reports error status: "+status.getErrorText()); - default: - ManifoldCF.sleep(1000L); - continue; - } - return; - } - throw new ManifoldCFException("ManifoldCF did not terminate in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - protected void waitJobDeleted(IJobManager jobManager, Long jobID, long maxTime) - throws ManifoldCFException, InterruptedException - { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() < startTime + maxTime) - { - JobStatus status = jobManager.getStatus(jobID); - if (status == null) - return; - ManifoldCF.sleep(1000L); - } - throw new ManifoldCFException("ManifoldCF did not delete in the allotted time of "+new Long(maxTime).toString()+" milliseconds"); - } - - }