lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1063640 - in /lucene/dev/branches/branch_3x/lucene/contrib/benchmark: CHANGES.txt build.xml src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
Date Wed, 26 Jan 2011 09:02:48 GMT
Author: shaie
Date: Wed Jan 26 09:02:48 2011
New Revision: 1063640

URL: http://svn.apache.org/viewvc?rev=1063640&view=rev
Log:
LUCENE-929: contrib/benchmark build doesn't handle checking if content is properly extracted
(3x)

Modified:
    lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt
    lucene/dev/branches/branch_3x/lucene/contrib/benchmark/build.xml
    lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java

Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt?rev=1063640&r1=1063639&r2=1063640&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt Wed Jan 26 09:02:48
2011
@@ -2,6 +2,11 @@ Lucene Benchmark Contrib Change Log
 
 The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
 
+01/26/2011
+  LUCENE-929: ExtractReuters first extracts to a tmp dir and then renames. That 
+  way, if a previous extract attempt failed, "ant extract-reuters" will still 
+  extract the files. (Shai Erera, Doron Cohen, Grant Ingersoll)
+  
 01/24/2011
   LUCENE-2885: Add WaitForMerges task (calls IndexWriter.waitForMerges()).
   (Mike McCandless)

Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/build.xml?rev=1063640&r1=1063639&r2=1063640&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/build.xml (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/build.xml Wed Jan 26 09:02:48 2011
@@ -82,7 +82,6 @@
 
     </target>
     <target name="extract-reuters" depends="check-files" unless="reuters.extracted">
-        <mkdir dir="${working.dir}/reuters-out"/>
         <java classname="org.apache.lucene.benchmark.utils.ExtractReuters" maxmemory="1024M"
fork="true">
             <classpath refid="run.classpath"/>
             <arg file="${working.dir}/reuters"/>

Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java?rev=1063640&r1=1063639&r2=1063640&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
Wed Jan 26 09:02:48 2011
@@ -1,4 +1,5 @@
 package org.apache.lucene.benchmark.utils;
+
 /**
  * Copyright 2005 The Apache Software Foundation
  *
@@ -15,7 +16,6 @@ package org.apache.lucene.benchmark.util
  * limitations under the License.
  */
 
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileFilter;
@@ -25,150 +25,123 @@ import java.io.IOException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-
 /**
- * Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline,
Body
+ * Split the Reuters SGML documents into Simple Text files containing: Title,
+ * Date, Dateline, Body
  */
-public class ExtractReuters
-{
-    private File reutersDir;
-    private File outputDir;
-    private static final String LINE_SEPARATOR = System.getProperty("line.separator");
-
-    public ExtractReuters(File reutersDir, File outputDir)
-    {
-        this.reutersDir = reutersDir;
-        this.outputDir = outputDir;
-        System.out.println("Deleting all files in " + outputDir);
-        File [] files = outputDir.listFiles();
-        for (int i = 0; i < files.length; i++)
-        {
-            files[i].delete();
-        }
-
+public class ExtractReuters {
+  private File reutersDir;
+  private File outputDir;
+  private static final String LINE_SEPARATOR = System.getProperty("line.separator");
+
+  public ExtractReuters(File reutersDir, File outputDir) {
+    this.reutersDir = reutersDir;
+    this.outputDir = outputDir;
+    System.out.println("Deleting all files in " + outputDir);
+    for (File f : outputDir.listFiles()) {
+      f.delete();
     }
+  }
 
-    public void extract()
-    {
-        File [] sgmFiles = reutersDir.listFiles(new FileFilter()
-        {
-            public boolean accept(File file)
-            {
-                return file.getName().endsWith(".sgm");
-            }
-        });
-        if (sgmFiles != null && sgmFiles.length > 0)
-        {
-            for (int i = 0; i < sgmFiles.length; i++)
-            {
-                File sgmFile = sgmFiles[i];
-                extractFile(sgmFile);
-            }
-        }
-        else
-        {
-            System.err.println("No .sgm files in " + reutersDir);
-        }
+  public void extract() {
+    File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
+      public boolean accept(File file) {
+        return file.getName().endsWith(".sgm");
+      }
+    });
+    if (sgmFiles != null && sgmFiles.length > 0) {
+      for (File sgmFile : sgmFiles) {
+        extractFile(sgmFile);
+      }
+    } else {
+      System.err.println("No .sgm files in " + reutersDir);
     }
+  }
 
-    Pattern EXTRACTION_PATTERN = Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
-
-    private static String[] META_CHARS
-            = {"&", "<", ">", "\"", "'"};
+  Pattern EXTRACTION_PATTERN = Pattern
+      .compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
 
-    private static String[] META_CHARS_SERIALIZATIONS
-            = {"&amp;", "&lt;", "&gt;", "&quot;", "&apos;"};
+  private static String[] META_CHARS = { "&", "<", ">", "\"", "'" };
 
-    /**
-     * Override if you wish to change what is extracted
-     *
-     * @param sgmFile
-     */
-    protected void extractFile(File sgmFile)
-    {
-        try
-        {
-            BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
-
-            StringBuilder buffer = new StringBuilder(1024);
-            StringBuilder outBuffer = new StringBuilder(1024);
-
-            String line = null;
-            int docNumber = 0;
-            while ((line = reader.readLine()) != null)
-            {
-                //when we see a closing reuters tag, flush the file
-
-                if (line.indexOf("</REUTERS") == -1) {
-                    //Replace the SGM escape sequences
-
-                    buffer.append(line).append(' ');//accumulate the strings for now, then
apply regular expression to get the pieces,
-                }
-                else
-                {
-                    //Extract the relevant pieces and write to a file in the output dir
-                    Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
-                    while (matcher.find())
-                    {
-                        for (int i = 1; i <= matcher.groupCount(); i++)
-                        {
-                            if (matcher.group(i) != null)
-                            {
-                                outBuffer.append(matcher.group(i));
-                            }
-                        }
-                        outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
-                    }
-                    String out = outBuffer.toString();
-                    for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++)
-                    {
-                        out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
-                    }
-                    File outFile = new File(outputDir, sgmFile.getName() + "-" + (docNumber++)
+ ".txt");
-                    //System.out.println("Writing " + outFile);
-                    FileWriter writer = new FileWriter(outFile);
-                    writer.write(out);
-                    writer.close();
-                    outBuffer.setLength(0);
-                    buffer.setLength(0);
-                }
+  private static String[] META_CHARS_SERIALIZATIONS = { "&amp;", "&lt;",
+      "&gt;", "&quot;", "&apos;" };
+
+  /**
+   * Override if you wish to change what is extracted
+   * 
+   * @param sgmFile
+   */
+  protected void extractFile(File sgmFile) {
+    try {
+      BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
+
+      StringBuilder buffer = new StringBuilder(1024);
+      StringBuilder outBuffer = new StringBuilder(1024);
+
+      String line = null;
+      int docNumber = 0;
+      while ((line = reader.readLine()) != null) {
+        // when we see a closing reuters tag, flush the file
+
+        if (line.indexOf("</REUTERS") == -1) {
+          // Replace the SGM escape sequences
+
+          buffer.append(line).append(' ');// accumulate the strings for now,
+                                          // then apply regular expression to
+                                          // get the pieces,
+        } else {
+          // Extract the relevant pieces and write to a file in the output dir
+          Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
+          while (matcher.find()) {
+            for (int i = 1; i <= matcher.groupCount(); i++) {
+              if (matcher.group(i) != null) {
+                outBuffer.append(matcher.group(i));
+              }
             }
-            reader.close();
-        }
-
-        catch (
-                IOException e
-                )
-
-        {
-            throw new RuntimeException(e);
-        }
+            outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
+          }
+          String out = outBuffer.toString();
+          for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
+            out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
+          }
+          File outFile = new File(outputDir, sgmFile.getName() + "-"
+              + (docNumber++) + ".txt");
+          // System.out.println("Writing " + outFile);
+          FileWriter writer = new FileWriter(outFile);
+          writer.write(out);
+          writer.close();
+          outBuffer.setLength(0);
+          buffer.setLength(0);
+        }
+      }
+      reader.close();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
+  }
 
-
-    public static void main(String[] args)
-    {
-        if (args.length != 2)
-        {
-            printUsage();
-        }
-        File reutersDir = new File(args[0]);
-
-        if (reutersDir.exists())
-        {
-            File outputDir = new File(args[1]);
-            outputDir.mkdirs();
-            ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
-            extractor.extract();
-        }
-        else
-        {
-            printUsage();
-        }
+  public static void main(String[] args) {
+    if (args.length != 2) {
+      printUsage();
     }
-
-    private static void printUsage()
-    {
-        System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters
<Path to Reuters SGM files> <Output Path>");
+    File reutersDir = new File(args[0]);
+    if (!reutersDir.exists()) {
+      printUsage();
+      return;
     }
+    
+    // First, extract to a tmp directory and only if everything succeeds, rename
+    // to output directory.
+    File outputDir = new File(args[1] + "-tmp");
+    outputDir.mkdirs();
+    ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
+    extractor.extract();
+    // Now rename to requested output dir
+    outputDir.renameTo(new File(args[1]));
+  }
+
+  private static void printUsage() {
+    System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters
<Path to Reuters SGM files> <Output Path>");
+  }
+  
 }



Mime
View raw message