lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dor...@apache.org
Subject svn commit: r1067359 - in /lucene/dev/trunk: ./ lucene/ lucene/src/test/org/apache/lucene/index/ lucene/src/test/org/apache/lucene/util/ modules/benchmark/ modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ modules/benchmark/src/java/...
Date Sat, 05 Feb 2011 00:35:10 GMT
Author: doronc
Date: Sat Feb  5 00:35:09 2011
New Revision: 1067359

URL: http://svn.apache.org/viewvc?rev=1067359&view=rev
Log:
LUCENE-1540: Improvements to contrib.benchmark for TREC collections - port/merge from 3x.

Added:
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
    lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip
      - copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip
Modified:
    lucene/dev/trunk/   (props changed)
    lucene/dev/trunk/lucene/   (props changed)
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
    lucene/dev/trunk/modules/benchmark/CHANGES.txt
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
    lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
    lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
(original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
Sat Feb  5 00:35:09 2011
@@ -17,20 +17,13 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import java.io.BufferedOutputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.Arrays;
-import java.util.Enumeration;
 import java.util.List;
 import java.util.Random;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
@@ -78,39 +71,6 @@ public class TestBackwardsCompatibility 
   }
   */
 
-  /* Unzips zipName --> dirName, removing dirName
-     first */
-  public void unzip(File zipName, String destDirName) throws IOException {
-
-    ZipFile zipFile = new ZipFile(zipName);
-
-    Enumeration<? extends ZipEntry> entries = zipFile.entries();
-
-    String dirName = fullDir(destDirName);
-
-    File fileDir = new File(dirName);
-    rmDir(destDirName);
-
-    fileDir.mkdir();
-
-    while (entries.hasMoreElements()) {
-      ZipEntry entry = entries.nextElement();
-
-      InputStream in = zipFile.getInputStream(entry);
-      OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(fileDir,
entry.getName())));
-
-      byte[] buffer = new byte[8192];
-      int len;
-      while((len = in.read(buffer)) >= 0) {
-        out.write(buffer, 0, len);
-      }
-
-      in.close();
-      out.close();
-    }
-
-    zipFile.close();
-  }
 /*
   public void testCreateCFS() throws IOException {
     String dirName = "testindex.cfs";
@@ -153,10 +113,9 @@ public class TestBackwardsCompatibility 
       if (VERBOSE) {
         System.out.println("TEST: index " + unsupportedNames[i]);
       }
-      unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), unsupportedNames[i]);
-
-      String fullPath = fullDir(unsupportedNames[i]);
-      Directory dir = newFSDirectory(new File(fullPath));
+      File oldIndxeDir = _TestUtil.getTempDir(unsupportedNames[i]);
+      _TestUtil.unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), oldIndxeDir);
+      Directory dir = newFSDirectory(oldIndxeDir);
 
       IndexReader reader = null;
       IndexWriter writer = null;
@@ -200,7 +159,7 @@ public class TestBackwardsCompatibility 
       assertTrue(bos.toString().contains(IndexFormatTooOldException.class.getName()));
 
       dir.close();
-      rmDir(unsupportedNames[i]);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
   
@@ -209,10 +168,9 @@ public class TestBackwardsCompatibility 
       if (VERBOSE) {
         System.out.println("\nTEST: index=" + oldNames[i]);
       }
-      unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-
-      String fullPath = fullDir(oldNames[i]);
-      Directory dir = newFSDirectory(new File(fullPath));
+      File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+      _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+      Directory dir = newFSDirectory(oldIndxeDir);
 
       IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
           TEST_VERSION_CURRENT, new MockAnalyzer()));
@@ -223,15 +181,15 @@ public class TestBackwardsCompatibility 
       _TestUtil.checkIndex(dir);
       
       dir.close();
-      rmDir(oldNames[i]);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
 
   public void testAddOldIndexes() throws IOException {
     for (String name : oldNames) {
-      unzip(getDataFile("index." + name + ".zip"), name);
-      String fullPath = fullDir(name);
-      Directory dir = newFSDirectory(new File(fullPath));
+      File oldIndxeDir = _TestUtil.getTempDir(name);
+      _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
+      Directory dir = newFSDirectory(oldIndxeDir);
 
       Directory targetDir = newDirectory();
       IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(
@@ -243,15 +201,15 @@ public class TestBackwardsCompatibility 
       
       dir.close();
       targetDir.close();
-      rmDir(name);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
 
   public void testAddOldIndexesReader() throws IOException {
     for (String name : oldNames) {
-      unzip(getDataFile("index." + name + ".zip"), name);
-      String fullPath = fullDir(name);
-      Directory dir = newFSDirectory(new File(fullPath));
+      File oldIndxeDir = _TestUtil.getTempDir(name);
+      _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
+      Directory dir = newFSDirectory(oldIndxeDir);
       IndexReader reader = IndexReader.open(dir);
       
       Directory targetDir = newDirectory();
@@ -265,23 +223,25 @@ public class TestBackwardsCompatibility 
       
       dir.close();
       targetDir.close();
-      rmDir(name);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
 
   public void testSearchOldIndex() throws IOException {
     for(int i=0;i<oldNames.length;i++) {
-      unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-      searchIndex(oldNames[i], oldNames[i]);
-      rmDir(oldNames[i]);
+      File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+      _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+      searchIndex(oldIndxeDir, oldNames[i]);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
 
   public void testIndexOldIndexNoAdds() throws IOException {
     for(int i=0;i<oldNames.length;i++) {
-      unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-      changeIndexNoAdds(random, oldNames[i]);
-      rmDir(oldNames[i]);
+      File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+      _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+      changeIndexNoAdds(random, oldIndxeDir);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
 
@@ -290,9 +250,10 @@ public class TestBackwardsCompatibility 
       if (VERBOSE) {
         System.out.println("TEST: oldName=" + oldNames[i]);
       }
-      unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-      changeIndexWithAdds(random, oldNames[i]);
-      rmDir(oldNames[i]);
+      File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+      _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+      changeIndexWithAdds(random, oldIndxeDir, oldNames[i]);
+      _TestUtil.rmDir(oldIndxeDir);
     }
   }
 
@@ -305,13 +266,11 @@ public class TestBackwardsCompatibility 
     }
   }
 
-  public void searchIndex(String dirName, String oldName) throws IOException {
+  public void searchIndex(File indexDir, String oldName) throws IOException {
     //QueryParser parser = new QueryParser("contents", new MockAnalyzer());
     //Query query = parser.parse("handle:1");
 
-    dirName = fullDir(dirName);
-
-    Directory dir = newFSDirectory(new File(dirName));
+    Directory dir = newFSDirectory(indexDir);
     IndexSearcher searcher = new IndexSearcher(dir, true);
     IndexReader reader = searcher.getIndexReader();
 
@@ -343,7 +302,7 @@ public class TestBackwardsCompatibility 
         }
 
         TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
-        assertNotNull("docID=" + i + " index=" + dirName, tfv);
+        assertNotNull("docID=" + i + " index=" + indexDir.getName(), tfv);
         assertTrue(tfv instanceof TermPositionVector);
       } else
         // Only ID 7 is deleted
@@ -376,11 +335,9 @@ public class TestBackwardsCompatibility 
     return v0 - v1;
   }
 
-  public void changeIndexWithAdds(Random random, String dirName) throws IOException {
-    String origDirName = dirName;
-    dirName = fullDir(dirName);
+  public void changeIndexWithAdds(Random random, File oldIndexDir, String origOldName) throws
IOException {
 
-    Directory dir = newFSDirectory(new File(dirName));
+    Directory dir = newFSDirectory(oldIndexDir);
     // open writer
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
     writer.setInfoStream(VERBOSE ? System.out : null);
@@ -391,7 +348,7 @@ public class TestBackwardsCompatibility 
 
     // make sure writer sees right total -- writer seems not to know about deletes in .del?
     final int expected;
-    if (compare(origDirName, "24") < 0) {
+    if (compare(origOldName, "24") < 0) {
       expected = 44;
     } else {
       expected = 45;
@@ -442,11 +399,9 @@ public class TestBackwardsCompatibility 
     dir.close();
   }
 
-  public void changeIndexNoAdds(Random random, String dirName) throws IOException {
-
-    dirName = fullDir(dirName);
+  public void changeIndexNoAdds(Random random, File oldIndexDir) throws IOException {
 
-    Directory dir = newFSDirectory(new File(dirName));
+    Directory dir = newFSDirectory(oldIndexDir);
 
     // make sure searching sees right # hits
     IndexSearcher searcher = new IndexSearcher(dir, true);
@@ -489,13 +444,12 @@ public class TestBackwardsCompatibility 
     dir.close();
   }
 
-  public void createIndex(Random random, String dirName, boolean doCFS) throws IOException
{
-
-    rmDir(dirName);
-
-    dirName = fullDir(dirName);
+  public File createIndex(Random random, String dirName, boolean doCFS) throws IOException
{
 
-    Directory dir = newFSDirectory(new File(dirName));
+    File indexDir = _TestUtil.getTempDir(dirName);
+    _TestUtil.rmDir(indexDir);
+    Directory dir = newFSDirectory(indexDir);
+    
     IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10);
     ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
     IndexWriter writer = new IndexWriter(dir, conf);
@@ -522,17 +476,21 @@ public class TestBackwardsCompatibility 
     // Set one norm so we get a .s0 file:
     reader.setNorm(21, "content", conf.getSimilarityProvider().get("content").encodeNormValue(1.5f));
     reader.close();
+    dir.close();
+    
+    return indexDir;
   }
 
   /* Verifies that the expected file names were produced */
 
   public void testExactFileNames() throws IOException {
 
-    String outputDir = "lucene.backwardscompat0.index";
-    rmDir(outputDir);
+    String outputDirName = "lucene.backwardscompat0.index";
+    File outputDir = _TestUtil.getTempDir(outputDirName);
+    _TestUtil.rmDir(outputDir);
 
     try {
-      Directory dir = newFSDirectory(new File(fullDir(outputDir)));
+      Directory dir = newFSDirectory(outputDir);
 
       LogMergePolicy mergePolicy = newLogMergePolicy(true, 10);
       mergePolicy.setNoCFSRatio(1); // This test expects all of its segments to be in CFS
@@ -595,7 +553,7 @@ public class TestBackwardsCompatibility 
       }
       dir.close();
     } finally {
-      rmDir(outputDir);
+      _TestUtil.rmDir(outputDir);
     }
   }
 
@@ -636,23 +594,6 @@ public class TestBackwardsCompatibility 
     writer.addDocument(doc);
   }
 
-  private void rmDir(String dir) throws IOException {
-    File fileDir = new File(fullDir(dir));
-    if (fileDir.exists()) {
-      File[] files = fileDir.listFiles();
-      if (files != null) {
-        for (int i = 0; i < files.length; i++) {
-          files[i].delete();
-        }
-      }
-      fileDir.delete();
-    }
-  }
-
-  public static String fullDir(String dirName) throws IOException {
-    return new File(TEMP_DIR, dirName).getCanonicalPath();
-  }
-
   private int countDocs(DocsEnum docs) throws IOException {
     int count = 0;
     while((docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
@@ -664,9 +605,9 @@ public class TestBackwardsCompatibility 
   // flex: test basics of TermsEnum api on non-flex index
   public void testNextIntoWrongField() throws Exception {
     for(int i=0;i<oldNames.length;i++) {
-      unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-      String fullPath = fullDir(oldNames[i]);
-      Directory dir = newFSDirectory(new File(fullPath));
+      File oldIndexDir = _TestUtil.getTempDir(oldNames[i]);
+    	_TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndexDir);
+      Directory dir = newFSDirectory(oldIndexDir);
       IndexReader r = IndexReader.open(dir);
       TermsEnum terms = MultiFields.getFields(r).terms("content").iterator();
       BytesRef t = terms.next();
@@ -703,16 +644,16 @@ public class TestBackwardsCompatibility 
 
       r.close();
       dir.close();
-      rmDir(oldNames[i]);
+      _TestUtil.rmDir(oldIndexDir);
     }
   }
   
   public void testNumericFields() throws Exception {
     for(int i=0;i<oldNames.length;i++) {
       
-      unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-      String fullPath = fullDir(oldNames[i]);
-      Directory dir = newFSDirectory(new File(fullPath));
+      File oldIndexDir = _TestUtil.getTempDir(oldNames[i]);
+      _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndexDir);
+      Directory dir = newFSDirectory(oldIndexDir);
       IndexSearcher searcher = new IndexSearcher(dir, true);
       
       for (int id=10; id<15; id++) {
@@ -747,7 +688,7 @@ public class TestBackwardsCompatibility 
       
       searcher.close();
       dir.close();
-      rmDir(oldNames[i]);
+      _TestUtil.rmDir(oldIndexDir);
     }
   }
 

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java Sat Feb  5 00:35:09
2011
@@ -17,13 +17,20 @@ package org.apache.lucene.util;
  * limitations under the License.
  */
 
+import java.io.BufferedOutputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.PrintStream;
+import java.util.Enumeration;
 import java.util.Random;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import org.junit.Assert;
 
@@ -61,6 +68,49 @@ public class _TestUtil {
     }
   }
 
+  /** 
+   * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first 
+   */
+  public static void unzip(File zipName, File destDir) throws IOException {
+    
+    ZipFile zipFile = new ZipFile(zipName);
+    
+    Enumeration<? extends ZipEntry> entries = zipFile.entries();
+    
+    rmDir(destDir);
+    
+    destDir.mkdir();
+    
+    while (entries.hasMoreElements()) {
+      ZipEntry entry = entries.nextElement();
+      
+      InputStream in = zipFile.getInputStream(entry);
+      File targetFile = new File(destDir, entry.getName());
+      if (entry.isDirectory()) {
+        // allow unzipping with directory structure
+        targetFile.mkdirs();
+      } else {
+        if (targetFile.getParentFile()!=null) {
+          // be on the safe side: do not rely on that directories are always extracted
+          // before their children (although this makes sense, but is it guaranteed?)
+          targetFile.getParentFile().mkdirs();   
+        }
+        OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
+        
+        byte[] buffer = new byte[8192];
+        int len;
+        while((len = in.read(buffer)) >= 0) {
+          out.write(buffer, 0, len);
+        }
+        
+        in.close();
+        out.close();
+      }
+    }
+    
+    zipFile.close();
+  }
+  
   public static void syncConcurrentMerges(IndexWriter writer) {
     syncConcurrentMerges(writer.getConfig().getMergeScheduler());
   }

Modified: lucene/dev/trunk/modules/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/CHANGES.txt?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/CHANGES.txt (original)
+++ lucene/dev/trunk/modules/benchmark/CHANGES.txt Sat Feb  5 00:35:09 2011
@@ -2,6 +2,12 @@ Lucene Benchmark Contrib Change Log
 
 The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
 
+02/05/2011
+  LUCENE-1540: Improvements to contrib.benchmark for TREC collections. 
+  ContentSource can now process plain text files, gzip files, and bzip2 files.
+  TREC doc parsing now handles the TREC gov2 collection and TREC disks 4&5-CR 
+  collection (both used by many TREC tasks). (Shai Erera, Doron Cohen)
+  
 01/26/2011
   LUCENE-929: ExtractReuters first extracts to a tmp dir and then renames. That 
   way, if a previous extract attempt failed, "ant extract-reuters" will still 

Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
Sat Feb  5 00:35:09 2011
@@ -56,11 +56,14 @@ import org.apache.lucene.benchmark.byTas
 public abstract class ContentSource {
   
   private static final int BZIP = 0;
-  private static final int OTHER = 1;
+  private static final int GZIP = 1;
+  private static final int OTHER = 2;
   private static final Map<String,Integer> extensionToType = new HashMap<String,Integer>();
   static {
     extensionToType.put(".bz2", Integer.valueOf(BZIP));
     extensionToType.put(".bzip", Integer.valueOf(BZIP));
+    extensionToType.put(".gz", Integer.valueOf(GZIP));
+    extensionToType.put(".gzip", Integer.valueOf(GZIP));
   }
   
   protected static final int BUFFER_SIZE = 1 << 16; // 64K
@@ -78,11 +81,13 @@ public abstract class ContentSource {
   
   private CompressorStreamFactory csFactory = new CompressorStreamFactory();
 
+  /** update count of bytes generated by this source */  
   protected final synchronized void addBytes(long numBytes) {
     bytesCount += numBytes;
     totalBytesCount += numBytes;
   }
   
+  /** update count of documents generated by this source */  
   protected final synchronized void addDoc() {
     ++docsCount;
     ++totalDocsCount;
@@ -130,21 +135,25 @@ public abstract class ContentSource {
         type = typeInt.intValue();
       }
     }
-    switch (type) {
-      case BZIP:
-        try {
+    
+    try {
+      switch (type) {
+        case BZIP:
           // According to BZip2CompressorInputStream's code, it reads the first 
           // two file header chars ('B' and 'Z'). It is important to wrap the
           // underlying input stream with a buffered one since
           // Bzip2CompressorInputStream uses the read() method exclusively.
           is = csFactory.createCompressorInputStream("bzip2", is);
-        } catch (CompressorException e) {
-          IOException ioe = new IOException(e.getMessage());
-          ioe.initCause(e);
-          throw ioe;
-        }
-        break;
-      default: // Do nothing, stay with FileInputStream
+          break;
+        case GZIP:
+          is = csFactory.createCompressorInputStream("gz", is);
+          break;
+        default: // Do nothing, stay with FileInputStream
+      }
+    } catch (CompressorException e) {
+      IOException ioe = new IOException(e.getMessage());
+      ioe.initCause(e);
+      throw ioe;
     }
     
     return is;

Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
Sat Feb  5 00:35:09 2011
@@ -29,11 +29,14 @@ import java.util.Properties;
  */
 public class DemoHTMLParser implements org.apache.lucene.benchmark.byTask.feeds.HTMLParser
{
 
-  public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat
dateFormat) throws IOException, InterruptedException {
+  public DocData parse(DocData docData, String name, Date date, String title, Reader reader,
DateFormat dateFormat) throws IOException, InterruptedException {
     org.apache.lucene.demo.html.HTMLParser p = new org.apache.lucene.demo.html.HTMLParser(reader);
     
     // title
-    String title = p.getTitle();
+    if (title==null) {
+      title = p.getTitle();
+    }
+    
     // properties 
     Properties props = p.getMetaTags(); 
     // body

Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
Sat Feb  5 00:35:09 2011
@@ -29,16 +29,18 @@ public interface HTMLParser {
 
   /**
    * Parse the input Reader and return DocData. 
-   * A provided name or date is used for the result, otherwise an attempt is 
-   * made to set them from the parsed data.
-   * @param dateFormat date formatter to use for extracting the date.   
-   * @param name name of the result doc data. If null, attempt to set by parsed data.
+   * The provided name,title,date are used for the result, unless when they're null, 
+   * in which case an attempt is made to set them from the parsed data.
+   * @param docData result reused
+   * @param name name of the result doc data.
    * @param date date of the result doc data. If null, attempt to set by parsed data.
-   * @param reader of html text to parse.
+   * @param title title of the result doc data. If null, attempt to set by parsed data.
+   * @param reader reader of html text to parse.
+   * @param dateFormat date formatter to use for extracting the date.   
    * @return Parsed doc data.
    * @throws IOException
    * @throws InterruptedException
    */
-  public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat
dateFormat) throws IOException, InterruptedException;
+  public DocData parse(DocData docData, String name, Date date, String title, Reader reader,
DateFormat dateFormat) throws IOException, InterruptedException;
 
 }

Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
Sat Feb  5 00:35:09 2011
@@ -19,8 +19,8 @@ package org.apache.lucene.benchmark.byTa
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.text.DateFormat;
@@ -29,8 +29,8 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.Locale;
-import java.util.zip.GZIPInputStream;
 
+import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
 import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.benchmark.byTask.utils.StringBuilderReader;
 import org.apache.lucene.util.ThreadInterruptedException;
@@ -46,8 +46,10 @@ import org.apache.lucene.util.ThreadInte
  * <li><b>docs.dir</b> - specifies the directory where the TREC files reside.
  * Can be set to a relative path if "work.dir" is also specified
  * (<b>default=trec</b>).
+ * <li><b>trec.doc.parser</b> - specifies the {@link TrecDocParser} class
to use for
+ * parsing the TREC documents content (<b>default=TrecGov2Parser</b>).
  * <li><b>html.parser</b> - specifies the {@link HTMLParser} class to use
for
- * parsing the TREC documents content (<b>default=DemoHTMLParser</b>).
+ * parsing the HTML parts of the TREC documents content (<b>default=DemoHTMLParser</b>).
  * <li><b>content.source.encoding</b> - if not specified, ISO-8859-1 is
used.
  * <li><b>content.source.excludeIteration</b> - if true, do not append
iteration number to docname
  * </ul>
@@ -59,22 +61,24 @@ public class TrecContentSource extends C
     ParsePosition pos;
   }
 
-  private static final String DATE = "Date: ";
-  private static final String DOCHDR = "<DOCHDR>";
-  private static final String TERMINATING_DOCHDR = "</DOCHDR>";
-  private static final String DOCNO = "<DOCNO>";
-  private static final String TERMINATING_DOCNO = "</DOCNO>";
-  private static final String DOC = "<DOC>";
-  private static final String TERMINATING_DOC = "</DOC>";
+  public static final String DOCNO = "<DOCNO>";
+  public static final String TERMINATING_DOCNO = "</DOCNO>";
+  public static final String DOC = "<DOC>";
+  public static final String TERMINATING_DOC = "</DOC>";
 
-  private static final String NEW_LINE = System.getProperty("line.separator");
+  /** separator between lines in the byffer */ 
+  public static final String NEW_LINE = System.getProperty("line.separator");
 
   private static final String DATE_FORMATS [] = {
-       "EEE, dd MMM yyyy kk:mm:ss z",	  // Tue, 09 Dec 2003 22:39:08 GMT
-       "EEE MMM dd kk:mm:ss yyyy z",  	// Tue Dec 09 16:45:08 2003 EST
-       "EEE, dd-MMM-':'y kk:mm:ss z", 	// Tue, 09 Dec 2003 22:39:08 GMT
-       "EEE, dd-MMM-yyy kk:mm:ss z", 	  // Tue, 09 Dec 2003 22:39:08 GMT
-       "EEE MMM dd kk:mm:ss yyyy",  	  // Tue Dec 09 16:45:08 2003
+       "EEE, dd MMM yyyy kk:mm:ss z",   // Tue, 09 Dec 2003 22:39:08 GMT
+       "EEE MMM dd kk:mm:ss yyyy z",    // Tue Dec 09 16:45:08 2003 EST
+       "EEE, dd-MMM-':'y kk:mm:ss z",   // Tue, 09 Dec 2003 22:39:08 GMT
+       "EEE, dd-MMM-yyy kk:mm:ss z",    // Tue, 09 Dec 2003 22:39:08 GMT
+       "EEE MMM dd kk:mm:ss yyyy",      // Tue Dec 09 16:45:08 2003
+       "dd MMM yyyy",                   // 1 March 1994
+       "MMM dd, yyyy",                  // February 3, 1994
+       "yyMMdd",                        // 910513
+       "hhmm z.z.z. MMM dd, yyyy",       // 0901 u.t.c. April 28, 1994
   };
 
   private ThreadLocal<DateFormatInfo> dateFormats = new ThreadLocal<DateFormatInfo>();
@@ -83,7 +87,7 @@ public class TrecContentSource extends C
   private File dataDir = null;
   private ArrayList<File> inputFiles = new ArrayList<File>();
   private int nextFile = 0;
-  private int rawDocSize;
+  private int rawDocSize = 0;
 
   // Use to synchronize threads on reading from the TREC documents.
   private Object lock = new Object();
@@ -92,7 +96,10 @@ public class TrecContentSource extends C
   BufferedReader reader;
   int iteration = 0;
   HTMLParser htmlParser;
+  
   private boolean excludeDocnameIteration;
+  private TrecDocParser trecDocParser = new TrecGov2Parser(); // default
+  ParsePathType currPathType; // not private for tests
   
   private DateFormatInfo getDateFormatInfo() {
     DateFormatInfo dfi = dateFormats.get();
@@ -118,7 +125,7 @@ public class TrecContentSource extends C
     return sb;
   }
   
-  private Reader getTrecDocReader(StringBuilder docBuffer) {
+  Reader getTrecDocReader(StringBuilder docBuffer) {
     StringBuilderReader r = trecDocReader.get();
     if (r == null) {
       r = new StringBuilderReader(docBuffer);
@@ -129,10 +136,21 @@ public class TrecContentSource extends C
     return r;
   }
 
-  // read until finding a line that starts with the specified prefix, or a terminating tag
has been found.
-  private void read(StringBuilder buf, String prefix, boolean collectMatchLine,
-                    boolean collectAll, String terminatingTag)
-      throws IOException, NoMoreDataException {
+  HTMLParser getHtmlParser() {
+    return htmlParser;
+  }
+  
+  /**
+   * Read until a line starting with the specified <code>lineStart</code>.
+   * @param buf buffer for collecting the data if so specified/ 
+   * @param lineStart line start to look for, must not be null.
+   * @param collectMatchLine whether to collect the matching line into <code>buffer</code>.
+   * @param collectAll whether to collect all lines into <code>buffer</code>.
+   * @throws IOException
+   * @throws NoMoreDataException
+   */
+   private void read(StringBuilder buf, String lineStart, 
+       boolean collectMatchLine, boolean collectAll) throws IOException, NoMoreDataException
{
     String sep = "";
     while (true) {
       String line = reader.readLine();
@@ -144,20 +162,12 @@ public class TrecContentSource extends C
 
       rawDocSize += line.length();
 
-      if (line.startsWith(prefix)) {
+      if (lineStart!=null && line.startsWith(lineStart)) {
         if (collectMatchLine) {
           buf.append(sep).append(line);
           sep = NEW_LINE;
         }
-        break;
-      }
-
-      if (terminatingTag != null && line.startsWith(terminatingTag)) {
-        // didn't find the prefix that was asked, but the terminating
-        // tag was found. set the length to 0 to signal no match was
-        // found.
-        buf.setLength(0);
-        break;
+        return;
       }
 
       if (collectAll) {
@@ -169,7 +179,7 @@ public class TrecContentSource extends C
   
   void openNextFile() throws NoMoreDataException, IOException {
     close();
-    int retries = 0;
+    currPathType = null;
     while (true) {
       if (nextFile >= inputFiles.size()) { 
         // exhausted files, start a new round, unless forever set to false.
@@ -184,13 +194,13 @@ public class TrecContentSource extends C
         System.out.println("opening: " + f + " length: " + f.length());
       }
       try {
-        GZIPInputStream zis = new GZIPInputStream(new FileInputStream(f), BUFFER_SIZE);
-        reader = new BufferedReader(new InputStreamReader(zis, encoding), BUFFER_SIZE);
+        InputStream inputStream = getInputStream(f); // support either gzip, bzip2, or regular
text file, by extension  
+        reader = new BufferedReader(new InputStreamReader(inputStream, encoding), BUFFER_SIZE);
+        currPathType = TrecDocParser.pathType(f);
         return;
       } catch (Exception e) {
-        retries++;
-        if (retries < 20 && verbose) {
-          System.out.println("Skipping 'bad' file " + f.getAbsolutePath() + "  #retries="
+ retries);
+        if (verbose) {
+          System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage());
           continue;
         }
         throw new NoMoreDataException();
@@ -198,7 +208,7 @@ public class TrecContentSource extends C
     }
   }
 
-  Date parseDate(String dateStr) {
+  public Date parseDate(String dateStr) {
     dateStr = dateStr.trim();
     DateFormatInfo dfi = getDateFormatInfo();
     for (int i = 0; i < dfi.dfs.length; i++) {
@@ -237,70 +247,47 @@ public class TrecContentSource extends C
 
   @Override
   public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException
{
-    String dateStr = null, name = null;
-    Reader r = null;
+    String name = null;
+    StringBuilder docBuf = getDocBuffer();
+    ParsePathType parsedPathType;
+    
     // protect reading from the TREC files by multiple threads. The rest of the
-    // method, i.e., parsing the content and returning the DocData can run
-    // unprotected.
+    // method, i.e., parsing the content and returning the DocData can run unprotected.
     synchronized (lock) {
       if (reader == null) {
         openNextFile();
       }
-
-      StringBuilder docBuf = getDocBuffer();
       
-      // 1. skip until doc start
+      // 1. skip until doc start - required for all TREC formats
       docBuf.setLength(0);
-      read(docBuf, DOC, false, false, null);
-
-      // 2. name
+      read(docBuf, DOC, false, false);
+      
+      // save parsedFile for passing trecDataParser after the sync block, in 
+      // case another thread will open another file in between.
+      parsedPathType = currPathType;
+      
+      // 2. name - required for all TREC formats
       docBuf.setLength(0);
-      read(docBuf, DOCNO, true, false, null);
+      read(docBuf, DOCNO, true, false);
       name = docBuf.substring(DOCNO.length(), docBuf.indexOf(TERMINATING_DOCNO,
-          DOCNO.length()));
-      if (!excludeDocnameIteration)
+          DOCNO.length())).trim();
+      
+      if (!excludeDocnameIteration) {
         name = name + "_" + iteration;
-
-      // 3. skip until doc header
-      docBuf.setLength(0);
-      read(docBuf, DOCHDR, false, false, null);
-
-      boolean findTerminatingDocHdr = false;
-
-      // 4. date - look for the date only until /DOCHDR
-      docBuf.setLength(0);
-      read(docBuf, DATE, true, false, TERMINATING_DOCHDR);
-      if (docBuf.length() != 0) {
-        // Date found.
-        dateStr = docBuf.substring(DATE.length());
-        findTerminatingDocHdr = true;
       }
 
-      // 5. skip until end of doc header
-      if (findTerminatingDocHdr) {
-        docBuf.setLength(0);
-        read(docBuf, TERMINATING_DOCHDR, false, false, null);
-      }
-
-      // 6. collect until end of doc
+      // 3. read all until end of doc
       docBuf.setLength(0);
-      read(docBuf, TERMINATING_DOC, false, true, null);
-      
-      // 7. Set up a Reader over the read content
-      r = getTrecDocReader(docBuf);
-      // Resetting the thread's reader means it will reuse the instance
-      // allocated as well as re-read from docBuf.
-      r.reset();
-      
-      // count char length of parsed html text (larger than the plain doc body text).
-      addBytes(docBuf.length()); 
+      read(docBuf, TERMINATING_DOC, false, true);
     }
+      
+    // count char length of text to be parsed (may be larger than the resulted plain doc
body text).
+    addBytes(docBuf.length()); 
 
     // This code segment relies on HtmlParser being thread safe. When we get 
     // here, everything else is already private to that thread, so we're safe.
-    Date date = dateStr != null ? parseDate(dateStr) : null;
     try {
-      docData = htmlParser.parse(docData, name, date, r, null);
+      docData = trecDocParser.parse(docData, name, this, docBuf, parsedPathType);
       addDoc();
     } catch (InterruptedException ie) {
       throw new ThreadInterruptedException(ie);
@@ -322,27 +309,40 @@ public class TrecContentSource extends C
   @Override
   public void setConfig(Config config) {
     super.setConfig(config);
+    // dirs
     File workDir = new File(config.get("work.dir", "work"));
     String d = config.get("docs.dir", "trec");
     dataDir = new File(d);
     if (!dataDir.isAbsolute()) {
       dataDir = new File(workDir, d);
     }
+    // files
     collectFiles(dataDir, inputFiles);
     if (inputFiles.size() == 0) {
       throw new IllegalArgumentException("No files in dataDir: " + dataDir);
     }
+    // trec doc parser
+    try {
+      String trecDocParserClassName = config.get("trec.doc.parser", "org.apache.lucene.benchmark.byTask.feeds.TrecGov2Parser");
+      trecDocParser = Class.forName(trecDocParserClassName).asSubclass(TrecDocParser.class).newInstance();
+    } catch (Exception e) {
+      // Should not get here. Throw runtime exception.
+      throw new RuntimeException(e);
+    }
+    // html parser
     try {
-      String parserClassName = config.get("html.parser",
+      String htmlParserClassName = config.get("html.parser",
           "org.apache.lucene.benchmark.byTask.feeds.DemoHTMLParser");
-      htmlParser = Class.forName(parserClassName).asSubclass(HTMLParser.class).newInstance();
+      htmlParser = Class.forName(htmlParserClassName).asSubclass(HTMLParser.class).newInstance();
     } catch (Exception e) {
       // Should not get here. Throw runtime exception.
       throw new RuntimeException(e);
     }
+    // encoding
     if (encoding == null) {
       encoding = "ISO-8859-1";
     }
+    // iteration exclusion in doc name 
     excludeDocnameIteration = config.get("content.source.excludeIteration", false);
   }
 

Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
Sat Feb  5 00:35:09 2011
@@ -158,8 +158,10 @@ public class StringBuilderReader extends
     synchronized (lock) {
       this.sb = sb;
       length = sb.length();
+      next = mark = 0;
     }
   }
+  
   @Override
   public long skip(long ns) throws IOException {
     synchronized (lock) {

Modified: lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
(original)
+++ lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
Sat Feb  5 00:35:09 2011
@@ -18,14 +18,20 @@ package org.apache.lucene.benchmark.byTa
  */
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.text.ParseException;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.HashSet;
+import java.util.Properties;
 
+import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
 import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 
 public class TrecContentSourceTest extends LuceneTestCase {
 
@@ -329,5 +335,62 @@ public class TrecContentSourceTest exten
     // Don't test that NoMoreDataException is thrown, since the forever flag is
     // turned on.
   }
+  
+  /** 
+   * Open a trec content source over a directory with files of all trec path types and all
+   * supported formats - bzip, gzip, txt. 
+   */
+  public void testTrecFeedDirAllTypes() throws Exception {
+    File dataDir =  _TestUtil.getTempDir("trecFeedAllTypes");
+    _TestUtil.unzip(getDataFile("trecdocs.zip"), dataDir);
+    TrecContentSource tcs = new TrecContentSource();
+    Properties props = new Properties();
+    props.setProperty("print.props", "false");
+    props.setProperty("content.source.verbose", "false");
+    props.setProperty("content.source.excludeIteration", "true");
+    props.setProperty("doc.maker.forever", "false");
+    props.setProperty("docs.dir", dataDir.getCanonicalPath().replace('\\','/')); 
+    props.setProperty("trec.doc.parser", TrecParserByPath.class.getName());
+    props.setProperty("content.source.forever", "false");
+    tcs.setConfig(new Config(props));
+    tcs.resetInputs();
+    DocData dd = new DocData();
+    int n = 0;
+    boolean gotExpectedException = false;
+    HashSet<ParsePathType> unseenTypes = new HashSet<ParsePathType>(Arrays.asList(ParsePathType.values()));
+    try {
+      while (n<100) { // arbiterary limit to prevent looping forever in case of test failure
+        dd = tcs.getNextDocData(dd);
+        ++n;
+        assertNotNull("doc data "+n+" should not be null!", dd);
+        unseenTypes.remove(tcs.currPathType);
+        switch(tcs.currPathType) {
+          case GOV2:
+            assertDocData(dd, "TEST-000", "TEST-000 title", "TEST-000 text", tcs.parseDate("Sun,
11 Jan 2009 08:00:00 GMT"));
+            break;
+          case FBIS:
+            assertDocData(dd, "TEST-001", "TEST-001 Title", "TEST-001 text", tcs.parseDate("1
January 1991"));
+            break;
+          case FR94:
+            // no title extraction in this source for now
+            assertDocData(dd, "TEST-002", null, "DEPARTMENT OF SOMETHING", tcs.parseDate("February
3, 1994"));
+            break;
+          case FT:
+            assertDocData(dd, "TEST-003", "Test-003 title", "Some pub text", tcs.parseDate("980424"));
+            break;
+          case LATIMES:
+            assertDocData(dd, "TEST-004", "Test-004 Title", "Some paragraph", tcs.parseDate("January
17, 1997, Sunday"));
+            break;
+          default:
+            assertTrue("Should never get here!", false);
+        }
+      }
+    } catch (NoMoreDataException e) {
+      gotExpectedException = true;
+    }
+    assertTrue("Should have gotten NoMoreDataException!", gotExpectedException);
+    assertEquals("Wrong numbre of documents created by osurce!",5,n);
+    assertTrue("Did not see all types!",unseenTypes.isEmpty());
+  }
 
 }



Mime
View raw message