manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1437592 - in /manifoldcf/trunk: ./ tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/
Date Wed, 23 Jan 2013 17:32:45 GMT
Author: kwright
Date: Wed Jan 23 17:32:45 2013
New Revision: 1437592

URL: http://svn.apache.org/viewvc?rev=1437592&view=rev
Log:
Solr tester, except for history check for failures

Added:
    manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
  (with props)
    manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
  (with props)
Modified:
    manifoldcf/trunk/build.xml
    manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java

Modified: manifoldcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/build.xml?rev=1437592&r1=1437591&r2=1437592&view=diff
==============================================================================
--- manifoldcf/trunk/build.xml (original)
+++ manifoldcf/trunk/build.xml Wed Jan 23 17:32:45 2013
@@ -2255,6 +2255,10 @@
         <ant dir="tests/jdbc" target="run-UI-derby"/>
     </target>
 
+    <target name="run-solr-tests-derby" depends="build-tests-framework,build-tests-filesystem-connector,build-tests-solr-connector,calculate-solr-tests-condition"
if="solr-tests.include">
+        <ant dir="tests/solr" target="run-derby"/>
+    </target>
+
     <target name="run-solr-UI-tests-derby" depends="build-tests-framework,build-tests-solr-connector,build-tests-filesystem-connector,calculate-solr-tests-condition"
if="solr-tests.include">
         <ant dir="tests/solr" target="run-UI-derby"/>
     </target>

Added: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java?rev=1437592&view=auto
==============================================================================
--- manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
(added)
+++ manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
Wed Jan 23 17:32:45 2013
@@ -0,0 +1,90 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.solr_tests;
+
+import java.io.*;
+import java.util.*;
+
+/** Helper class for local file manipulation */
+public class FileHelper
+{
+
+  protected static void createDirectory(File f)
+    throws Exception
+  {
+    if (f.mkdirs() == false)
+      throw new Exception("Failed to create directory "+f.toString());
+  }
+  
+  protected static void removeDirectory(File f)
+    throws Exception
+  {
+    File[] files = f.listFiles();
+    if (files != null)
+    {
+      int i = 0;
+      while (i < files.length)
+      {
+        File subfile = files[i++];
+        if (subfile.isDirectory())
+          removeDirectory(subfile);
+        else
+          subfile.delete();
+      }
+    }
+    f.delete();
+  }
+  
+  protected static void createFile(File f, String contents)
+    throws Exception
+  {
+    OutputStream os = new FileOutputStream(f);
+    try
+    {
+      Writer w = new OutputStreamWriter(os,"utf-8");
+      try
+      {
+        w.write(contents);
+      }
+      finally
+      {
+        w.flush();
+      }
+    }
+    finally
+    {
+      os.close();
+    }
+  }
+  
+  protected static void removeFile(File f)
+    throws Exception
+  {
+    if (f.delete() == false)
+      throw new Exception("Failed to delete file "+f.toString());
+  }
+  
+  protected static void changeFile(File f, String newContents)
+    throws Exception
+  {
+    removeFile(f);
+    createFile(f,newContents);
+  }
+  
+}

Propchange: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java?rev=1437592&r1=1437591&r2=1437592&view=diff
==============================================================================
--- manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
(original)
+++ manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
Wed Jan 23 17:32:45 2013
@@ -16,7 +16,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package org.apache.manifoldcf.rss_tests;
+package org.apache.manifoldcf.solr_tests;
 
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.server.Server;

Added: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java?rev=1437592&view=auto
==============================================================================
--- manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
(added)
+++ manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
Wed Jan 23 17:32:45 2013
@@ -0,0 +1,176 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.solr_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+import org.apache.manifoldcf.agents.output.solr.SolrConfig;
+
+/** This is a 100 document crawl */
+public class SolrTester
+{
+  protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+  
+  public SolrTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance)
+  {
+    this.instance = instance;
+  }
+  
+  public void setupTestArea()
+    throws Exception
+  {
+    File f = new File("testdata");
+    FileHelper.removeDirectory(f);
+    FileHelper.createDirectory(f);
+    // Create the test data files.
+    String baseFileName = "testdata/";
+    int i0 = 0;
+    while (i0 < 10)
+    {
+      String fileName0 = baseFileName + "/dir-" + i0;
+      FileHelper.createDirectory(new File(fileName0));
+      int i1 = 0;
+      while (i1 < 10)
+      {
+        String fileName1 = fileName0 + "/dir-" + i1;
+        FileHelper.createDirectory(new File(fileName1));
+        int i2 = 0;
+        while (i2 < 10)
+        {
+          String fileName2 = fileName1 + "/file-"+i2;
+          FileHelper.createFile(new File(fileName2),"Test file "+i0+":"+i1+":"+i2);
+          i2++;
+        }
+        i1++;
+      }
+      i0++;
+    }
+    System.err.println("Done generating files");
+  }
+  
+  public void teardownTestArea()
+    throws Exception
+  {
+    System.err.println("Removing generated files");
+    File f = new File("testdata");
+    FileHelper.removeDirectory(f);
+  }
+  
+  public void executeTest()
+    throws Exception
+  {
+    // Hey, we were able to install the file system connector etc.
+    // Now, create a local test job and run it.
+    IThreadContext tc = ThreadContextFactory.make();
+      
+    // Create a basic file system connection, and save it.
+    IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+    IRepositoryConnection conn = mgr.create();
+    conn.setName("File Connection");
+    conn.setDescription("File Connection");
+    conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+    conn.setMaxConnections(100);
+    // Now, save
+    mgr.save(conn);
+      
+    // Create a basic null output connection, and save it.
+    IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+    IOutputConnection outputConn = outputMgr.create();
+    outputConn.setName("Solr Connection");
+    outputConn.setDescription("Solr Connection");
+    outputConn.setClassName("org.apache.manifoldcf.agents.output.solr.SolrConnector");
+    outputConn.setMaxConnections(10);
+    // Set the connection parameters
+    ConfigParams configParams = outputConn.getConfigParams();
+    configParams.setParameter(SolrConfig.PARAM_PROTOCOL,SolrConfig.PROTOCOL_TYPE_HTTP);
+    configParams.setParameter(SolrConfig.PARAM_SERVER,"localhost");
+    configParams.setParameter(SolrConfig.PARAM_PORT,"8188");
+    configParams.setParameter(SolrConfig.PARAM_WEBAPPNAME,"solr");
+    configParams.setParameter(SolrConfig.PARAM_UPDATEPATH,"/update/extract");
+    configParams.setParameter(SolrConfig.PARAM_REMOVEPATH,"/update");
+    configParams.setParameter(SolrConfig.PARAM_STATUSPATH,"/admin/ping");
+    configParams.setParameter(SolrConfig.PARAM_IDFIELD,"id");
+    // Now, save
+    outputMgr.save(outputConn);
+
+    // Create a job.
+    IJobManager jobManager = JobManagerFactory.make(tc);
+    IJobDescription job = jobManager.createJob();
+    job.setDescription("Test Job");
+    job.setConnectionName("File Connection");
+    job.setOutputConnectionName("Solr Connection");
+    job.setType(job.TYPE_SPECIFIED);
+    job.setStartMethod(job.START_DISABLE);
+    job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
+      
+    // Now, set up the document specification.
+    DocumentSpecification ds = job.getSpecification();
+    // Crawl everything underneath the 'testdata' area
+    File testDataFile = new File("testdata").getCanonicalFile();
+    if (!testDataFile.exists())
+      throw new ManifoldCFException("Test data area not found!  Looking in "+testDataFile.toString());
+    if (!testDataFile.isDirectory())
+      throw new ManifoldCFException("Test data area not a directory!  Looking in "+testDataFile.toString());
+    SpecificationNode sn = new SpecificationNode("startpoint");
+    sn.setAttribute("path",testDataFile.toString());
+    SpecificationNode n = new SpecificationNode("include");
+    n.setAttribute("type","file");
+    n.setAttribute("match","*");
+    sn.addChild(sn.getChildCount(),n);
+    n = new SpecificationNode("include");
+    n.setAttribute("type","directory");
+    n.setAttribute("match","*");
+    sn.addChild(sn.getChildCount(),n);
+    ds.addChild(ds.getChildCount(),sn);
+      
+    // Set up the output specification.
+    OutputSpecification os = job.getOutputSpecification();
+    // Solr output specification is not needed
+      
+    // Save the job.
+    jobManager.save(job);
+
+    // Now, start the job, and wait until it completes.
+    long startTime = System.currentTimeMillis();
+    jobManager.manualStart(job.getID());
+    instance.waitJobInactiveNative(jobManager,job.getID(),18000L);
+    System.err.println("Crawl required "+new Long(System.currentTimeMillis()-startTime).toString()+"
milliseconds");
+
+    // Look in the connection history for anything other than an OK
+    // MHL
+    
+    // Check to be sure we actually processed the right number of documents.
+    JobStatus status = jobManager.getStatus(job.getID());
+    if (status.getDocumentsProcessed() != 111)
+      throw new ManifoldCFException("Wrong number of documents processed - expected 111,
saw "+new Long(status.getDocumentsProcessed()).toString());
+      
+    // Now, delete the job.
+    jobManager.deleteJob(job.getID());
+    instance.waitJobDeletedNative(jobManager,job.getID(),18000L);
+      
+    // Cleanup is automatic by the base class, so we can feel free to leave jobs and connections
lying around.
+  }
+  
+}

Propchange: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
------------------------------------------------------------------------------
    svn:keywords = Id



Mime
View raw message