manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1610852 - in /manifoldcf/trunk: ./ framework/ framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/
Date Tue, 15 Jul 2014 21:06:52 GMT
Author: kwright
Date: Tue Jul 15 21:06:52 2014
New Revision: 1610852

URL: http://svn.apache.org/r1610852
Log:
Fix for CONNECTORS-992

Added:
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
  (with props)
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
  (with props)
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
  (with props)
Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/build.xml

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1610852&r1=1610851&r2=1610852&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Tue Jul 15 21:06:52 2014
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 1.7-dev =====================
 
+CONNECTORS-992: Add a test to exercise ServiceInterruption within
+a connector.
+(Karl Wright)
+
 CONNECTORS-990: Revamp IRepositoryConnector API to no longer separate
 getDocumentVersions() and processDocuments().  This modification basically
 pushes responsibility to determine changes to the repository connector.

Modified: manifoldcf/trunk/framework/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/build.xml?rev=1610852&r1=1610851&r2=1610852&view=diff
==============================================================================
--- manifoldcf/trunk/framework/build.xml (original)
+++ manifoldcf/trunk/framework/build.xml Tue Jul 15 21:06:52 2014
@@ -1657,6 +1657,7 @@
             <formatter type="brief" usefile="false"/>
 
             <test name="org.apache.manifoldcf.crawler.tests.SchedulerHSQLDBTest" todir="test-output"/>
+            <test name="org.apache.manifoldcf.crawler.tests.InterruptionHSQLDBTest" todir="test-output"/>
 
         </junit>
     </target>

Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java?rev=1610852&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
(added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
Tue Jul 15 21:06:52 2014
@@ -0,0 +1,128 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.agents.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test of service interruptions */
+public class InterruptionHSQLDBTest extends ConnectorBaseHSQLDB
+{
+  protected final ManifoldCFInstance mcfInstance;
+  protected InterruptionTester tester;
+
+  public InterruptionHSQLDBTest()
+  {
+    super();
+    mcfInstance = new ManifoldCFInstance("A",false,false);
+    tester = new InterruptionTester(mcfInstance);
+  }
+  
+  @Override
+  protected String[] getConnectorClasses()
+  {
+    return new String[]{"org.apache.manifoldcf.crawler.tests.InterruptionRepositoryConnector"};
+  }
+  
+  @Override
+  protected String[] getConnectorNames()
+  {
+    return new String[]{"InterruptionConnector"};
+  }
+
+  @Override
+  protected String[] getOutputClasses()
+  {
+    return new String[]{"org.apache.manifoldcf.agents.tests.TestingOutputConnector"};
+  }
+  
+  @Override
+  protected String[] getOutputNames()
+  {
+    return new String[]{"NullOutput"};
+  }
+
+  @Test
+  public void interruptionTestRun()
+    throws Exception
+  {
+    tester.executeTest();
+  }
+  
+  @Before
+  public void setUp()
+    throws Exception
+  {
+    initializeSystem();
+    try
+    {
+      localReset();
+    }
+    catch (Exception e)
+    {
+      System.out.println("Warning: Preclean failed: "+e.getMessage());
+    }
+    try
+    {
+      localSetUp();
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void cleanUp()
+    throws Exception
+  {
+    Exception currentException = null;
+    // Last, shut down the web applications.
+    // If this is done too soon it closes the database before the rest of the cleanup happens.
+    try
+    {
+      mcfInstance.unload();
+    }
+    catch (Exception e)
+    {
+      if (currentException == null)
+        currentException = e;
+    }
+    try
+    {
+      localCleanUp();
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+    if (currentException != null)
+      throw currentException;
+    cleanupSystem();
+  }
+  
+
+}

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java?rev=1610852&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
(added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
Tue Jul 15 21:06:52 2014
@@ -0,0 +1,103 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+
+/** Connector class to be used by general integration tests that need documents */
+public class InterruptionRepositoryConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
+{
+  public InterruptionRepositoryConnector()
+  {
+  }
+
+  @Override
+  public void addSeedDocuments(ISeedingActivity activities, DocumentSpecification spec,
+    long startTime, long endTime, int jobMode)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    String docCount = "10";
+    for (int i = 0; i < spec.getChildCount(); i++)
+    {
+      SpecificationNode sn = spec.getChild(i);
+      if (sn.getType().equals("documentcount"))
+        docCount = sn.getAttributeValue("count");
+    }
+    int count = Integer.parseInt(docCount);
+    
+    for (int i = 0; i < count; i++)
+    {
+      String doc = "test"+i+".txt";
+      activities.addSeedDocument(doc,null);
+    }
+  }
+  
+  @Override
+  public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions,
IVersionActivity activities,
+    DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    String[] rval = new String[documentIdentifiers.length];
+    for (int i = 0; i < rval.length; i++)
+    {
+      rval[i] = "";
+    }
+    return rval;
+  }
+
+  @Override
+  public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity
activities,
+    DocumentSpecification spec, boolean[] scanOnly, int jobMode)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    for (int i = 0; i < documentIdentifiers.length; i++)
+    {
+      String documentIdentifier = documentIdentifiers[i];
+      String version = versions[i];
+      if (!scanOnly[i])
+      {
+        if (documentIdentifier.equals("test0.txt"))
+        {
+          // This will emulate one particular document failing (and being skipped)
+          long currentTime = System.currentTimeMillis();
+          throw new ServiceInterruption("Pretending there's a service interruption",
+            null,currentTime+1000L,currentTime+5000L,10,false);
+        }
+        RepositoryDocument rd = new RepositoryDocument();
+        byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
+        rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
+        try
+        {
+          activities.ingestDocumentWithException(documentIdentifier,version,"http://"+documentIdentifier,rd);
+        }
+        catch (IOException e)
+        {
+          throw new RuntimeException("Shouldn't be seeing IOException from binary array input
stream: "+e.getMessage(),e);
+        }
+      }
+    }
+  }
+
+}

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java?rev=1610852&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
(added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
Tue Jul 15 21:06:52 2014
@@ -0,0 +1,99 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a test whether we can handle service interruptions */
+public class InterruptionTester
+{
+  protected final ManifoldCFInstance instance;
+  
+  public InterruptionTester(ManifoldCFInstance instance)
+  {
+    this.instance = instance;
+  }
+  
+  public void executeTest()
+    throws Exception
+  {
+    instance.start();
+    
+    // Hey, we were able to install the file system connector etc.
+    // Now, create a local test job and run it.
+    IThreadContext tc = ThreadContextFactory.make();
+      
+    // Create a basic file system connection, and save it.
+    IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+    IRepositoryConnection conn = mgr.create();
+    conn.setName("InterruptionTest Connection");
+    conn.setDescription("InterruptionTest Connection");
+    conn.setClassName("org.apache.manifoldcf.crawler.tests.InterruptionRepositoryConnector");
+    conn.setMaxConnections(100);
+    // Now, save
+    mgr.save(conn);
+      
+    // Create a basic null output connection, and save it.
+    IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+    IOutputConnection outputConn = outputMgr.create();
+    outputConn.setName("Null Connection");
+    outputConn.setDescription("Null Connection");
+    outputConn.setClassName("org.apache.manifoldcf.agents.tests.TestingOutputConnector");
+    outputConn.setMaxConnections(100);
+    // Now, save
+    outputMgr.save(outputConn);
+
+    // Create a job.
+    IJobManager jobManager = JobManagerFactory.make(tc);
+    IJobDescription job = jobManager.createJob();
+    job.setDescription("Test Job");
+    job.setConnectionName("InterruptionTest Connection");
+    job.addPipelineStage(-1,true,"Null Connection","");
+    //job.setOutputConnectionName("Null Connection");
+    job.setType(job.TYPE_SPECIFIED);
+    job.setStartMethod(job.START_DISABLE);
+    job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      
+    // Save the job.
+    jobManager.save(job);
+
+    // Now, start the job, and wait until it is running.
+    jobManager.manualStart(job.getID());
+    instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+    
+    // Wait for the job to become inactive.  The time should not exceed 10 seconds for the
actual crawl.
+    instance.waitJobInactiveNative(jobManager,job.getID(),30000L);
+    // The document will be skipped in the end.
+    if (jobManager.getStatus(job.getID()).getDocumentsProcessed() != 9)
+      throw new Exception("Expected 9 documents, saw "+jobManager.getStatus(job.getID()).getDocumentsProcessed());
+    
+    // Now, delete the job.
+    jobManager.deleteJob(job.getID());
+    instance.waitJobDeletedNative(jobManager,job.getID(),30000L);
+
+    // Shut down instance2
+    instance.stop();
+  }
+}

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
------------------------------------------------------------------------------
    svn:keywords = Id



Mime
View raw message