lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r476359 [2/2] - in /lucene/java/trunk: ./ docs/ src/java/org/apache/lucene/index/ src/java/org/apache/lucene/store/ src/test/org/apache/lucene/index/ src/test/org/apache/lucene/store/ xdocs/
Date Fri, 17 Nov 2006 23:18:49 GMT
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java Fri Nov 17 15:18:47 2006
@@ -33,6 +33,7 @@
  */
 class SegmentReader extends IndexReader {
   private String segment;
+  private SegmentInfo si;
 
   FieldInfos fieldInfos;
   private FieldsReader fieldsReader;
@@ -64,22 +65,24 @@
     private boolean dirty;
     private int number;
 
-    private void reWrite() throws IOException {
+    private void reWrite(SegmentInfo si) throws IOException {
       // NOTE: norms are re-written in regular directory, not cfs
-      IndexOutput out = directory().createOutput(segment + ".tmp");
+
+      String oldFileName = si.getNormFileName(this.number);
+      if (oldFileName != null) {
+        // Mark this file for deletion.  Note that we don't
+        // actually try to delete it until the new segments files is
+        // successfully written:
+        deleter.addPendingFile(oldFileName);
+      }
+
+      si.advanceNormGen(this.number);
+      IndexOutput out = directory().createOutput(si.getNormFileName(this.number));
       try {
         out.writeBytes(bytes, maxDoc());
       } finally {
         out.close();
       }
-      String fileName;
-      if(cfsReader == null)
-          fileName = segment + ".f" + number;
-      else{
-          // use a different file name if we have compound format
-          fileName = segment + ".s" + number;
-      }
-      directory().renameFile(segment + ".tmp", fileName);
       this.dirty = false;
     }
   }
@@ -131,57 +134,94 @@
     return instance;
   }
 
-   private void initialize(SegmentInfo si) throws IOException {
+  private void initialize(SegmentInfo si) throws IOException {
     segment = si.name;
+    this.si = si;
 
-    // Use compound file directory for some files, if it exists
-    Directory cfsDir = directory();
-    if (directory().fileExists(segment + ".cfs")) {
-      cfsReader = new CompoundFileReader(directory(), segment + ".cfs");
-      cfsDir = cfsReader;
-    }
-
-    // No compound file exists - use the multi-file format
-    fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
-    fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
-
-    tis = new TermInfosReader(cfsDir, segment, fieldInfos);
+    boolean success = false;
 
-    // NOTE: the bitvector is stored using the regular directory, not cfs
-    if (hasDeletions(si))
-      deletedDocs = new BitVector(directory(), segment + ".del");
+    try {
+      // Use compound file directory for some files, if it exists
+      Directory cfsDir = directory();
+      if (si.getUseCompoundFile()) {
+        cfsReader = new CompoundFileReader(directory(), segment + ".cfs");
+        cfsDir = cfsReader;
+      }
+
+      // No compound file exists - use the multi-file format
+      fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
+      fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
+
+      tis = new TermInfosReader(cfsDir, segment, fieldInfos);
+      
+      // NOTE: the bitvector is stored using the regular directory, not cfs
+      if (hasDeletions(si)) {
+        deletedDocs = new BitVector(directory(), si.getDelFileName());
+      }
+
+      // make sure that all index files have been read or are kept open
+      // so that if an index update removes them we'll still have them
+      freqStream = cfsDir.openInput(segment + ".frq");
+      proxStream = cfsDir.openInput(segment + ".prx");
+      openNorms(cfsDir);
 
-    // make sure that all index files have been read or are kept open
-    // so that if an index update removes them we'll still have them
-    freqStream = cfsDir.openInput(segment + ".frq");
-    proxStream = cfsDir.openInput(segment + ".prx");
-    openNorms(cfsDir);
+      if (fieldInfos.hasVectors()) { // open term vector files only as needed
+        termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
+      }
+      success = true;
+    } finally {
 
-    if (fieldInfos.hasVectors()) { // open term vector files only as needed
-      termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
+      // With lock-less commits, it's entirely possible (and
+      // fine) to hit a FileNotFound exception above.  In
+      // this case, we want to explicitly close any subset
+      // of things that were opened so that we don't have to
+      // wait for a GC to do so.
+      if (!success) {
+        doClose();
+      }
     }
   }
 
-   protected void finalize() {
+  protected void finalize() {
      // patch for pre-1.4.2 JVMs, whose ThreadLocals leak
      termVectorsLocal.set(null);
      super.finalize();
-   }
+  }
 
   protected void doCommit() throws IOException {
     if (deletedDocsDirty) {               // re-write deleted
-      deletedDocs.write(directory(), segment + ".tmp");
-      directory().renameFile(segment + ".tmp", segment + ".del");
-    }
-    if(undeleteAll && directory().fileExists(segment + ".del")){
-      directory().deleteFile(segment + ".del");
+      String oldDelFileName = si.getDelFileName();
+      if (oldDelFileName != null) {
+        // Mark this file for deletion.  Note that we don't
+        // actually try to delete it until the new segments files is
+        // successfully written:
+        deleter.addPendingFile(oldDelFileName);
+      }
+
+      si.advanceDelGen();
+
+      // We can write directly to the actual name (vs to a
+      // .tmp & renaming it) because the file is not live
+      // until segments file is written:
+      deletedDocs.write(directory(), si.getDelFileName());
+    }
+    if (undeleteAll && si.hasDeletions()) {
+      String oldDelFileName = si.getDelFileName();
+      if (oldDelFileName != null) {
+        // Mark this file for deletion.  Note that we don't
+        // actually try to delete it until the new segments files is
+        // successfully written:
+        deleter.addPendingFile(oldDelFileName);
+      }
+      si.clearDelGen();
     }
     if (normsDirty) {               // re-write norms
+      si.setNumField(fieldInfos.size());
       Enumeration values = norms.elements();
       while (values.hasMoreElements()) {
         Norm norm = (Norm) values.nextElement();
         if (norm.dirty) {
-          norm.reWrite();
+          norm.reWrite(si);
         }
       }
     }
@@ -191,8 +231,12 @@
   }
 
   protected void doClose() throws IOException {
-    fieldsReader.close();
-    tis.close();
+    if (fieldsReader != null) {
+      fieldsReader.close();
+    }
+    if (tis != null) {
+      tis.close();
+    }
 
     if (freqStream != null)
       freqStream.close();
@@ -209,27 +253,19 @@
   }
 
   static boolean hasDeletions(SegmentInfo si) throws IOException {
-    return si.dir.fileExists(si.name + ".del");
+    return si.hasDeletions();
   }
 
   public boolean hasDeletions() {
     return deletedDocs != null;
   }
 
-
   static boolean usesCompoundFile(SegmentInfo si) throws IOException {
-    return si.dir.fileExists(si.name + ".cfs");
+    return si.getUseCompoundFile();
   }
 
   static boolean hasSeparateNorms(SegmentInfo si) throws IOException {
-    String[] result = si.dir.list();
-    String pattern = si.name + ".s";
-    int patternLength = pattern.length();
-    for(int i = 0; i < result.length; i++){
-      if(result[i].startsWith(pattern) && Character.isDigit(result[i].charAt(patternLength)))
-        return true;
-    }
-    return false;
+    return si.hasSeparateNorms();
   }
 
   protected void doDelete(int docNum) {
@@ -249,23 +285,27 @@
   Vector files() throws IOException {
     Vector files = new Vector(16);
 
-    for (int i = 0; i < IndexFileNames.INDEX_EXTENSIONS.length; i++) {
-      String name = segment + "." + IndexFileNames.INDEX_EXTENSIONS[i];
-      if (directory().fileExists(name))
+    if (si.getUseCompoundFile()) {
+      String name = segment + ".cfs";
+      if (directory().fileExists(name)) {
         files.addElement(name);
+      }
+    } else {
+      for (int i = 0; i < IndexFileNames.INDEX_EXTENSIONS.length; i++) {
+        String name = segment + "." + IndexFileNames.INDEX_EXTENSIONS[i];
+        if (directory().fileExists(name))
+          files.addElement(name);
+      }
+    }
+
+    if (si.hasDeletions()) {
+      files.addElement(si.getDelFileName());
     }
 
     for (int i = 0; i < fieldInfos.size(); i++) {
-      FieldInfo fi = fieldInfos.fieldInfo(i);
-      if (fi.isIndexed  && !fi.omitNorms){
-        String name;
-        if(cfsReader == null)
-            name = segment + ".f" + i;
-        else
-            name = segment + ".s" + i;
-        if (directory().fileExists(name))
+      String name = si.getNormFileName(i);
+      if (name != null && directory().fileExists(name))
             files.addElement(name);
-      }
     }
     return files;
   }
@@ -380,7 +420,6 @@
   protected synchronized byte[] getNorms(String field) throws IOException {
     Norm norm = (Norm) norms.get(field);
     if (norm == null) return null;  // not indexed, or norms not stored
-
     if (norm.bytes == null) {                     // value not yet read
       byte[] bytes = new byte[maxDoc()];
       norms(field, bytes, 0);
@@ -436,12 +475,10 @@
     for (int i = 0; i < fieldInfos.size(); i++) {
       FieldInfo fi = fieldInfos.fieldInfo(i);
       if (fi.isIndexed && !fi.omitNorms) {
-        // look first if there are separate norms in compound format
-        String fileName = segment + ".s" + fi.number;
         Directory d = directory();
-        if(!d.fileExists(fileName)){
-            fileName = segment + ".f" + fi.number;
-            d = cfsDir;
+        String fileName = si.getNormFileName(fi.number);
+        if (!si.hasSeparateNorms(fi.number)) {
+          d = cfsDir;
         }
         norms.put(fi.name, new Norm(d.openInput(fileName), fi.number));
       }

Modified: lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/FSDirectory.java Fri Nov 17 15:18:47 2006
@@ -128,7 +128,7 @@
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(String path, boolean create)
       throws IOException {
-    return getDirectory(path, create, null);
+    return getDirectory(new File(path), create, null, true);
   }
 
   /** Returns the directory instance for the named location, using the
@@ -144,9 +144,15 @@
    *        locking implementation.
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(String path, boolean create,
+                                         LockFactory lockFactory, boolean doRemoveOldFiles)
+      throws IOException {
+    return getDirectory(new File(path), create, lockFactory, doRemoveOldFiles);
+  }
+
+  public static FSDirectory getDirectory(String path, boolean create,
                                          LockFactory lockFactory)
       throws IOException {
-    return getDirectory(new File(path), create, lockFactory);
+    return getDirectory(new File(path), create, lockFactory, true);
   }
 
   /** Returns the directory instance for the named location.
@@ -158,9 +164,9 @@
    * @param file the path to the directory.
    * @param create if true, create, or erase any existing contents.
    * @return the FSDirectory for the named file.  */
-  public static FSDirectory getDirectory(File file, boolean create)
+  public static FSDirectory getDirectory(File file, boolean create, boolean doRemoveOldFiles)
     throws IOException {
-    return getDirectory(file, create, null);
+    return getDirectory(file, create, null, doRemoveOldFiles);
   }
 
   /** Returns the directory instance for the named location, using the
@@ -176,7 +182,7 @@
    *        locking implementation.
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(File file, boolean create,
-                                         LockFactory lockFactory)
+                                         LockFactory lockFactory, boolean doRemoveOldFiles)
     throws IOException {
     file = new File(file.getCanonicalPath());
     FSDirectory dir;
@@ -188,7 +194,7 @@
         } catch (Exception e) {
           throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e);
         }
-        dir.init(file, create, lockFactory);
+        dir.init(file, create, lockFactory, doRemoveOldFiles);
         DIRECTORIES.put(file, dir);
       } else {
 
@@ -199,7 +205,7 @@
         }
 
         if (create) {
-          dir.create();
+          dir.create(doRemoveOldFiles);
         }
       }
     }
@@ -209,23 +215,35 @@
     return dir;
   }
 
+  public static FSDirectory getDirectory(File file, boolean create,
+                                         LockFactory lockFactory)
+    throws IOException
+  {
+    return getDirectory(file, create, lockFactory, true);
+  }
+
+  public static FSDirectory getDirectory(File file, boolean create)
+    throws IOException {
+    return getDirectory(file, create, true);
+  }
+
   private File directory = null;
   private int refCount;
 
   protected FSDirectory() {};                     // permit subclassing
 
-  private void init(File path, boolean create) throws IOException {
+  private void init(File path, boolean create, boolean doRemoveOldFiles) throws IOException {
     directory = path;
 
     if (create) {
-      create();
+      create(doRemoveOldFiles);
     }
 
     if (!directory.isDirectory())
       throw new IOException(path + " not a directory");
   }
 
-  private void init(File path, boolean create, LockFactory lockFactory) throws IOException {
+  private void init(File path, boolean create, LockFactory lockFactory, boolean doRemoveOldFiles) throws IOException {
 
     // Set up lockFactory with cascaded defaults: if an instance was passed in,
     // use that; else if locks are disabled, use NoLockFactory; else if the
@@ -280,10 +298,10 @@
 
     setLockFactory(lockFactory);
 
-    init(path, create);
+    init(path, create, doRemoveOldFiles);
   }
 
-  private synchronized void create() throws IOException {
+  private synchronized void create(boolean doRemoveOldFiles) throws IOException {
     if (!directory.exists())
       if (!directory.mkdirs())
         throw new IOException("Cannot create directory: " + directory);
@@ -291,13 +309,15 @@
     if (!directory.isDirectory())
       throw new IOException(directory + " not a directory");
 
-    String[] files = directory.list(new IndexFileNameFilter());            // clear old files
-    if (files == null)
-      throw new IOException("Cannot read directory " + directory.getAbsolutePath());
-    for (int i = 0; i < files.length; i++) {
-      File file = new File(directory, files[i]);
-      if (!file.delete())
-        throw new IOException("Cannot delete " + file);
+    if (doRemoveOldFiles) {
+      String[] files = directory.list(IndexFileNameFilter.getFilter());            // clear old files
+      if (files == null)
+        throw new IOException("Cannot read directory " + directory.getAbsolutePath());
+      for (int i = 0; i < files.length; i++) {
+        File file = new File(directory, files[i]);
+        if (!file.delete())
+          throw new IOException("Cannot delete " + file);
+      }
     }
 
     lockFactory.clearAllLocks();
@@ -305,7 +325,7 @@
 
   /** Returns an array of strings, one for each Lucene index file in the directory. */
   public String[] list() {
-    return directory.list(new IndexFileNameFilter());
+    return directory.list(IndexFileNameFilter.getFilter());
   }
 
   /** Returns true iff a file with the given name exists. */

Modified: lucene/java/trunk/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/RAMDirectory.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/RAMDirectory.java Fri Nov 17 15:18:47 2006
@@ -18,6 +18,7 @@
  */
 
 import java.io.IOException;
+import java.io.FileNotFoundException;
 import java.io.File;
 import java.io.Serializable;
 import java.util.Hashtable;
@@ -105,7 +106,7 @@
   }
 
   /** Returns an array of strings, one for each file in the directory. */
-  public final String[] list() {
+  public synchronized final String[] list() {
     String[] result = new String[files.size()];
     int i = 0;
     Enumeration names = files.keys();
@@ -129,7 +130,7 @@
   /** Set the modified time of an existing file to now. */
   public void touchFile(String name) {
 //     final boolean MONITOR = false;
-    
+
     RAMFile file = (RAMFile)files.get(name);
     long ts2, ts1 = System.currentTimeMillis();
     do {
@@ -175,8 +176,11 @@
   }
 
   /** Returns a stream reading an existing file. */
-  public final IndexInput openInput(String name) {
+  public final IndexInput openInput(String name) throws IOException {
     RAMFile file = (RAMFile)files.get(name);
+    if (file == null) {
+      throw new FileNotFoundException(name);
+    }
     return new RAMInputStream(file);
   }
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java Fri Nov 17 15:18:47 2006
@@ -32,6 +32,7 @@
 
 import java.util.Collection;
 import java.io.IOException;
+import java.io.FileNotFoundException;
 import java.io.File;
 
 public class TestIndexReader extends TestCase
@@ -222,6 +223,11 @@
         assertEquals("deleted count", 100, deleted);
         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
+
+        // open a 2nd reader to make sure first reader can
+        // commit its changes (.del) while second reader
+        // is open:
+        IndexReader reader2 = IndexReader.open(dir);
         reader.close();
 
         // CREATE A NEW READER and re-test
@@ -231,10 +237,73 @@
         reader.close();
     }
 
+    // Make sure you can set norms & commit even if a reader
+    // is open against the index:
+    public void testWritingNorms() throws IOException
+    {
+        String tempDir = System.getProperty("tempDir");
+        if (tempDir == null)
+            throw new IOException("tempDir undefined, cannot run test");
+
+        File indexDir = new File(tempDir, "lucenetestnormwriter");
+        Directory dir = FSDirectory.getDirectory(indexDir, true);
+        IndexWriter writer = null;
+        IndexReader reader = null;
+        Term searchTerm = new Term("content", "aaa");
+
+        //  add 1 documents with term : aaa
+        writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        addDoc(writer, searchTerm.text());
+        writer.close();
+
+        //  now open reader & set norm for doc 0
+        reader = IndexReader.open(dir);
+        reader.setNorm(0, "content", (float) 2.0);
+
+        // we should be holding the write lock now:
+        assertTrue("locked", IndexReader.isLocked(dir));
+
+        reader.commit();
+
+        // we should not be holding the write lock now:
+        assertTrue("not locked", !IndexReader.isLocked(dir));
+
+        // open a 2nd reader:
+        IndexReader reader2 = IndexReader.open(dir);
+
+        // set norm again for doc 0
+        reader.setNorm(0, "content", (float) 3.0);
+        assertTrue("locked", IndexReader.isLocked(dir));
+
+        reader.close();
+
+        // we should not be holding the write lock now:
+        assertTrue("not locked", !IndexReader.isLocked(dir));
+
+        reader2.close();
+        dir.close();
+
+        rmDir(indexDir);
+    }
+
 
     public void testDeleteReaderWriterConflictUnoptimized() throws IOException{
       deleteReaderWriterConflict(false);
     }
+
+    public void testOpenEmptyDirectory() throws IOException{
+      String dirName = "test.empty";
+      File fileDirName = new File(dirName);
+      if (!fileDirName.exists()) {
+        fileDirName.mkdir();
+      }
+      try {
+        IndexReader reader = IndexReader.open(fileDirName);
+        fail("opening IndexReader on empty directory failed to produce FileNotFoundException");
+      } catch (FileNotFoundException e) {
+        // GOOD
+      }
+    }
     
     public void testDeleteReaderWriterConflictOptimized() throws IOException{
         deleteReaderWriterConflict(true);
@@ -368,12 +437,36 @@
       assertFalse(IndexReader.isLocked(dir));		// reader only, no lock
       long version = IndexReader.lastModified(dir);
       reader.close();
-      // modify index and check version has been incremented:
+      // modify index and check version has been
+      // incremented:
       writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
       addDocumentWithFields(writer);
       writer.close();
       reader = IndexReader.open(dir);
-      assertTrue(version < IndexReader.getCurrentVersion(dir));
+      assertTrue("old lastModified is " + version + "; new lastModified is " + IndexReader.lastModified(dir), version <= IndexReader.lastModified(dir));
+      reader.close();
+    }
+
+    public void testVersion() throws IOException {
+      assertFalse(IndexReader.indexExists("there_is_no_such_index"));
+      Directory dir = new RAMDirectory();
+      assertFalse(IndexReader.indexExists(dir));
+      IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      addDocumentWithFields(writer);
+      assertTrue(IndexReader.isLocked(dir));		// writer open, so dir is locked
+      writer.close();
+      assertTrue(IndexReader.indexExists(dir));
+      IndexReader reader = IndexReader.open(dir);
+      assertFalse(IndexReader.isLocked(dir));		// reader only, no lock
+      long version = IndexReader.getCurrentVersion(dir);
+      reader.close();
+      // modify index and check version has been
+      // incremented:
+      writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      addDocumentWithFields(writer);
+      writer.close();
+      reader = IndexReader.open(dir);
+      assertTrue("old version is " + version + "; new version is " + IndexReader.getCurrentVersion(dir), version < IndexReader.getCurrentVersion(dir));
       reader.close();
     }
 
@@ -412,6 +505,40 @@
       reader.close();
     }
 
+    public void testUndeleteAllAfterClose() throws IOException {
+      Directory dir = new RAMDirectory();
+      IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      addDocumentWithFields(writer);
+      addDocumentWithFields(writer);
+      writer.close();
+      IndexReader reader = IndexReader.open(dir);
+      reader.deleteDocument(0);
+      reader.deleteDocument(1);
+      reader.close();
+      reader = IndexReader.open(dir);
+      reader.undeleteAll();
+      assertEquals(2, reader.numDocs());	// nothing has really been deleted thanks to undeleteAll()
+      reader.close();
+    }
+
+    public void testUndeleteAllAfterCloseThenReopen() throws IOException {
+      Directory dir = new RAMDirectory();
+      IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      addDocumentWithFields(writer);
+      addDocumentWithFields(writer);
+      writer.close();
+      IndexReader reader = IndexReader.open(dir);
+      reader.deleteDocument(0);
+      reader.deleteDocument(1);
+      reader.close();
+      reader = IndexReader.open(dir);
+      reader.undeleteAll();
+      reader.close();
+      reader = IndexReader.open(dir);
+      assertEquals(2, reader.numDocs());	// nothing has really been deleted thanks to undeleteAll()
+      reader.close();
+    }
+
     public void testDeleteReaderReaderConflictUnoptimized() throws IOException{
       deleteReaderReaderConflict(false);
     }
@@ -561,5 +688,12 @@
         Document doc = new Document();
         doc.add(new Field("content", value, Field.Store.NO, Field.Index.TOKENIZED));
         writer.addDocument(doc);
+    }
+    private void rmDir(File dir) {
+        File[] files = dir.listFiles();
+        for (int i = 0; i < files.length; i++) {
+            files[i].delete();
+        }
+        dir.delete();
     }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Nov 17 15:18:47 2006
@@ -1,6 +1,7 @@
 package org.apache.lucene.index;
 
 import java.io.IOException;
+import java.io.File;
 
 import junit.framework.TestCase;
 
@@ -10,7 +11,10 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 
 
 /**
@@ -28,14 +32,11 @@
         int i;
 
         IndexWriter.setDefaultWriteLockTimeout(2000);
-        IndexWriter.setDefaultCommitLockTimeout(2000);
         assertEquals(2000, IndexWriter.getDefaultWriteLockTimeout());
-        assertEquals(2000, IndexWriter.getDefaultCommitLockTimeout());
 
         writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
 
         IndexWriter.setDefaultWriteLockTimeout(1000);
-        IndexWriter.setDefaultCommitLockTimeout(1000);
 
         // add 100 documents
         for (i = 0; i < 100; i++) {
@@ -72,6 +73,12 @@
         assertEquals(60, reader.maxDoc());
         assertEquals(60, reader.numDocs());
         reader.close();
+
+        // make sure opening a new index for create over
+        // this existing one works correctly:
+        writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        assertEquals(0, writer.docCount());
+        writer.close();
     }
 
     private void addDoc(IndexWriter writer) throws IOException
@@ -79,5 +86,193 @@
         Document doc = new Document();
         doc.add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
         writer.addDocument(doc);
+    }
+
+    // Make sure we can open an index for create even when a
+    // reader holds it open (this fails pre lock-less
+    // commits on windows):
+    public void testCreateWithReader() throws IOException {
+        String tempDir = System.getProperty("java.io.tmpdir");
+        if (tempDir == null)
+            throw new IOException("java.io.tmpdir undefined, cannot run test");
+        File indexDir = new File(tempDir, "lucenetestindexwriter");
+        Directory dir = FSDirectory.getDirectory(indexDir, true);
+
+        // add one document & close writer
+        IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        addDoc(writer);
+        writer.close();
+
+        // now open reader:
+        IndexReader reader = IndexReader.open(dir);
+        assertEquals("should be one document", reader.numDocs(), 1);
+
+        // now open index for create:
+        writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        assertEquals("should be zero documents", writer.docCount(), 0);
+        addDoc(writer);
+        writer.close();
+
+        assertEquals("should be one document", reader.numDocs(), 1);
+        IndexReader reader2 = IndexReader.open(dir);
+        assertEquals("should be one document", reader2.numDocs(), 1);
+        reader.close();
+        reader2.close();
+        rmDir(indexDir);
+    }
+
+    // Simulate a writer that crashed while writing segments
+    // file: make sure we can still open the index (ie,
+    // gracefully fallback to the previous segments file),
+    // and that we can add to the index:
+    public void testSimulatedCrashedWriter() throws IOException {
+        Directory dir = new RAMDirectory();
+
+        IndexWriter writer = null;
+
+        writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+        // add 100 documents
+        for (int i = 0; i < 100; i++) {
+            addDoc(writer);
+        }
+
+        // close
+        writer.close();
+
+        long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+        assertTrue("segment generation should be > 1 but got " + gen, gen > 1);
+
+        // Make the next segments file, with last byte
+        // missing, to simulate a writer that crashed while
+        // writing segments file:
+        String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
+        String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                   "",
+                                                                   1+gen);
+        IndexInput in = dir.openInput(fileNameIn);
+        IndexOutput out = dir.createOutput(fileNameOut);
+        long length = in.length();
+        for(int i=0;i<length-1;i++) {
+          out.writeByte(in.readByte());
+        }
+        in.close();
+        out.close();
+
+        IndexReader reader = null;
+        try {
+          reader = IndexReader.open(dir);
+        } catch (Exception e) {
+          fail("reader failed to open on a crashed index");
+        }
+        reader.close();
+
+        try {
+          writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        } catch (Exception e) {
+          fail("writer failed to open on a crashed index");
+        }
+
+        // add 100 documents
+        for (int i = 0; i < 100; i++) {
+            addDoc(writer);
+        }
+
+        // close
+        writer.close();
+    }
+
+    // Simulate a corrupt index by removing last byte of
+    // latest segments file and make sure we get an
+    // IOException trying to open the index:
+    public void testSimulatedCorruptIndex1() throws IOException {
+        Directory dir = new RAMDirectory();
+
+        IndexWriter writer = null;
+
+        writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+        // add 100 documents
+        for (int i = 0; i < 100; i++) {
+            addDoc(writer);
+        }
+
+        // close
+        writer.close();
+
+        long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+        assertTrue("segment generation should be > 1 but got " + gen, gen > 1);
+
+        String fileNameIn = SegmentInfos.getCurrentSegmentFileName(dir);
+        String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                   "",
+                                                                   1+gen);
+        IndexInput in = dir.openInput(fileNameIn);
+        IndexOutput out = dir.createOutput(fileNameOut);
+        long length = in.length();
+        for(int i=0;i<length-1;i++) {
+          out.writeByte(in.readByte());
+        }
+        in.close();
+        out.close();
+        dir.deleteFile(fileNameIn);
+
+        IndexReader reader = null;
+        try {
+          reader = IndexReader.open(dir);
+          fail("reader did not hit IOException on opening a corrupt index");
+        } catch (Exception e) {
+        }
+        if (reader != null) {
+          reader.close();
+        }
+    }
+
+    // Simulate a corrupt index by removing one of the cfs
+    // files and make sure we get an IOException trying to
+    // open the index:
+    public void testSimulatedCorruptIndex2() throws IOException {
+        Directory dir = new RAMDirectory();
+
+        IndexWriter writer = null;
+
+        writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+        // add 100 documents
+        for (int i = 0; i < 100; i++) {
+            addDoc(writer);
+        }
+
+        // close
+        writer.close();
+
+        long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+        assertTrue("segment generation should be > 1 but got " + gen, gen > 1);
+
+        String[] files = dir.list();
+        for(int i=0;i<files.length;i++) {
+          if (files[i].endsWith(".cfs")) {
+            dir.deleteFile(files[i]);
+            break;
+          }
+        }
+
+        IndexReader reader = null;
+        try {
+          reader = IndexReader.open(dir);
+          fail("reader did not hit IOException on opening a corrupt index");
+        } catch (Exception e) {
+        }
+        if (reader != null) {
+          reader.close();
+        }
+    }
+
+    private void rmDir(File dir) {
+        File[] files = dir.listFiles();
+        for (int i = 0; i < files.length; i++) {
+            files[i].delete();
+        }
+        dir.delete();
     }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiReader.java Fri Nov 17 15:18:47 2006
@@ -80,6 +80,21 @@
     assertEquals( 1, reader.numDocs() );
     reader.undeleteAll();
     assertEquals( 2, reader.numDocs() );
+
+    // Ensure undeleteAll survives commit/close/reopen:
+    reader.commit();
+    reader.close();
+    sis.read(dir);
+    reader = new MultiReader(dir, sis, false, readers);
+    assertEquals( 2, reader.numDocs() );
+
+    reader.deleteDocument(0);
+    assertEquals( 1, reader.numDocs() );
+    reader.commit();
+    reader.close();
+    sis.read(dir);
+    reader = new MultiReader(dir, sis, false, readers);
+    assertEquals( 1, reader.numDocs() );
   }
         
   

Added: lucene/java/trunk/src/test/org/apache/lucene/index/index.prelockless.cfs.zip
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/index.prelockless.cfs.zip?view=auto&rev=476359
==============================================================================
Binary file - no diff available.

Propchange: lucene/java/trunk/src/test/org/apache/lucene/index/index.prelockless.cfs.zip
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: lucene/java/trunk/src/test/org/apache/lucene/index/index.prelockless.nocfs.zip
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/index.prelockless.nocfs.zip?view=auto&rev=476359
==============================================================================
Binary file - no diff available.

Propchange: lucene/java/trunk/src/test/org/apache/lucene/index/index.prelockless.nocfs.zip
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/store/TestLockFactory.java Fri Nov 17 15:18:47 2006
@@ -58,9 +58,9 @@
 
         // Both write lock and commit lock should have been created:
         assertEquals("# of unique locks created (after instantiating IndexWriter)",
-                     2, lf.locksCreated.size());
-        assertTrue("# calls to makeLock <= 2 (after instantiating IndexWriter)",
-                   lf.makeLockCount > 2);
+                     1, lf.locksCreated.size());
+        assertTrue("# calls to makeLock is 0 (after instantiating IndexWriter)",
+                   lf.makeLockCount >= 1);
         
         for(Enumeration e = lf.locksCreated.keys(); e.hasMoreElements();) {
             String lockName = (String) e.nextElement();
@@ -90,6 +90,7 @@
         try {
             writer2 = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
         } catch (Exception e) {
+            e.printStackTrace(System.out);
             fail("Should not have hit an IOException with no locking");
         }
 
@@ -234,6 +235,7 @@
         try {
             writer2 = new IndexWriter(indexDirName, new WhitespaceAnalyzer(), false);
         } catch (IOException e) {
+            e.printStackTrace(System.out);
             fail("Should not have hit an IOException with locking disabled");
         }
 
@@ -266,6 +268,7 @@
         try {
             fs2 = FSDirectory.getDirectory(indexDirName, true, lf);
         } catch (IOException e) {
+            e.printStackTrace(System.out);
             fail("Should not have hit an IOException because LockFactory instances are the same");
         }
 
@@ -294,7 +297,6 @@
 
     public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException {
         FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, true, lockFactory);
-        // fs1.setLockFactory(NoLockFactory.getNoLockFactory());
 
         // First create a 1 doc index:
         IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true);
@@ -405,6 +407,7 @@
                     hitException = true;
                     System.out.println("Stress Test Index Writer: creation hit unexpected exception: " + e.toString());
                     e.printStackTrace(System.out);
+                    break;
                 }
                 if (writer != null) {
                     try {
@@ -413,6 +416,7 @@
                         hitException = true;
                         System.out.println("Stress Test Index Writer: addDoc hit unexpected exception: " + e.toString());
                         e.printStackTrace(System.out);
+                        break;
                     }
                     try {
                         writer.close();
@@ -420,6 +424,7 @@
                         hitException = true;
                         System.out.println("Stress Test Index Writer: close hit unexpected exception: " + e.toString());
                         e.printStackTrace(System.out);
+                        break;
                     }
                     writer = null;
                 }
@@ -446,6 +451,7 @@
                     hitException = true;
                     System.out.println("Stress Test Index Searcher: create hit unexpected exception: " + e.toString());
                     e.printStackTrace(System.out);
+                    break;
                 }
                 if (searcher != null) {
                     Hits hits = null;
@@ -455,6 +461,7 @@
                         hitException = true;
                         System.out.println("Stress Test Index Searcher: search hit unexpected exception: " + e.toString());
                         e.printStackTrace(System.out);
+                        break;
                     }
                     // System.out.println(hits.length() + " total results");
                     try {
@@ -463,6 +470,7 @@
                         hitException = true;
                         System.out.println("Stress Test Index Searcher: close hit unexpected exception: " + e.toString());
                         e.printStackTrace(System.out);
+                        break;
                     }
                     searcher = null;
                 }

Modified: lucene/java/trunk/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/java/trunk/xdocs/fileformats.xml?view=diff&rev=476359&r1=476358&r2=476359
==============================================================================
--- lucene/java/trunk/xdocs/fileformats.xml (original)
+++ lucene/java/trunk/xdocs/fileformats.xml Fri Nov 17 15:18:47 2006
@@ -14,7 +14,7 @@
 
             <p>
                 This document defines the index file formats used
-                in Lucene version 2.0.  If you are using a different
+                in Lucene version 2.1.  If you are using a different
 		version of Lucene, please consult the copy of
 		<code>docs/fileformats.html</code> that was distributed
 		with the version you are using.
@@ -43,6 +43,18 @@
                 describing how file formats have changed from prior versions.
             </p>
 
+            <p>
+	        In version 2.1, the file format was changed to allow
+	        lock-less commits (ie, no more commit lock).  The
+	        change is fully backwards compatible: you can open a
+	        pre-2.1 index for searching or adding/deleting of
+	        docs.  When the new segments file is saved
+	        (committed), it will be written in the new file format
+	        (meaning no specific "upgrade" process is needed).
+	        But note that once a commit has occurred, pre-2.1
+	        Lucene will not be able to read the index.
+	    </p>
+
         </section>
 
         <section name="Definitions">
@@ -260,6 +272,18 @@
                 required.
             </p>
 
+	    <p>
+	        As of version 2.1 (lock-less commits), file names are
+	        never re-used (there is one exception, "segments.gen",
+	        see below).  That is, when any file is saved to the
+	        Directory it is given a never before used filename.
+	        This is achieved using a simple generations approach.
+	        For example, the first segments file is segments_1,
+	        then segments_2, etc.  The generation is a sequential
+	        long integer represented in alpha-numeric (base 36)
+	        form.
+            </p>
+
         </section>
 
         <section name="Primitive Types">
@@ -696,22 +720,48 @@
 
                 <p>
                     The active segments in the index are stored in the
-                    segment info file.  An index only has
-                    a single file in this format, and it is named "segments".
-                    This lists each segment by name, and also contains the size of each
-                    segment.
-                </p>
+                    segment info file, <tt>segments_N</tt>.  There may
+                    be one or more <tt>segments_N</tt> files in the
+                    index; however, the one with the largest
+                    generation is the active one (when older
+                    segments_N files are present it's because they
+                    temporarily cannot be deleted, or, a writer is in
+                    the process of committing). This file lists each
+                    segment by name, has details about the separate
+                    norms and deletion files, and also contains the
+                    size of each segment.
+                </p>
+
+		<p>
+		    As of 2.1, there is also a file
+		    <tt>segments.gen</tt>.  This file contains the
+		    current generation (the <tt>_N</tt> in
+		    <tt>segments_N</tt>) of the index.  This is
+		    used only as a fallback in case the current
+		    generation cannot be accurately determined by
+		    directory listing alone (as is the case for some
+		    NFS clients with time-based directory cache
+		    expiraation).  This file simply contains an Int32
+		    version header (SegmentInfos.FORMAT_LOCKLESS =
+		    -2), followed by the generation recorded as Int64,
+		    written twice.
+		</p>
 
                 <p>
+		<b>Pre-2.1:</b>
                     Segments    --&gt; Format, Version, NameCounter, SegCount, &lt;SegName, SegSize&gt;<sup>SegCount</sup>
                 </p>
+		<p>
+		<b>2.1 and above:</b>
+                    Segments    --&gt; Format, Version, NameCounter, SegCount, &lt;SegName, SegSize, DelGen, NumField, NormGen<sup>NumField</sup> &gt;<sup>SegCount</sup>, IsCompoundFile
+                </p>
 
                 <p>
-                    Format, NameCounter, SegCount, SegSize    --&gt; UInt32
+                    Format, NameCounter, SegCount, SegSize, NumField    --&gt; Int32
                 </p>
 
                 <p>
-                    Version --&gt; UInt64
+                    Version, DelGen, NormGen --&gt; Int64
                 </p>
 
                 <p>
@@ -719,7 +769,11 @@
                 </p>
 
                 <p>
-                    Format is -1 in Lucene 1.4.
+                    IsCompoundFile    --&gt; Int8
+                </p>
+
+                <p>
+                    Format is -1 as of Lucene 1.4 and -2 as of Lucene 2.1.
                 </p>
 
                 <p>
@@ -740,65 +794,79 @@
                     SegSize is the number of documents contained in the segment index.
                 </p>
 
+                <p>
+                    DelGen is the generation count of the separate
+                    deletes file.  If this is -1, there are no
+                    separate deletes.  If it is 0, this is a pre-2.1
+                    segment and you must check filesystem for the
+                    existence of _X.del.  Anything above zero means
+                    there are separate deletes (_X_N.del).
+                </p>
+
+                <p>
+                    NumField is the size of the array for NormGen, or
+                    -1 if there are no NormGens stored.
+                </p>
+
+                <p>
+                    NormGen records the generation of the separate
+                    norms files.  If NumField is -1, there are no
+                    normGens stored and they are all assumed to be 0
+                    when the segment file was written pre-2.1 and all
+                    assumed to be -1 when the segments file is 2.1 or
+                    above.  The generation then has the same meaning
+                    as delGen (above).
+                </p>
+
+                <p>
+                    IsCompoundFile records whether the segment is
+                    written as a compound file or not.  If this is -1,
+                    the segment is not a compound file.  If it is 1,
+                    the segment is a compound file.  Else it is 0,
+                    which means we check filesystem to see if _X.cfs
+                    exists.
+                </p>
+
 
             </subsection>
 
-            <subsection name="Lock Files">
+            <subsection name="Lock File">
 
                 <p>
-                    Several files are used to indicate that another
-                    process is using an index.  Note that these files are not
+                    A write lock is used to indicate that another
+                    process is writing to the index.  Note that this file is not
                     stored in the index directory itself, but rather in the
                     system's temporary directory, as indicated in the Java
                     system property "java.io.tmpdir".
                 </p>
 
-                <ul>
-                    <li>
-                        <p>
-                            When a file named "commit.lock"
-                            is present, a process is currently re-writing the "segments"
-                            file and deleting outdated segment index files, or a process is
-                            reading the "segments"
-                            file and opening the files of the segments it names.  This lock file
-                            prevents files from being deleted by another process after a process
-                            has read the "segments"
-                            file but before it has managed to open all of the files of the
-                            segments named therein.
-                        </p>
-                    </li>
+                <p>
+                    The write lock is named "XXXX-write.lock" where
+                    XXXX is typically a unique prefix computed by the
+                    directory path to the index.  When this file is
+                    present, a process is currently adding documents
+                    to an index, or removing files from that index.
+                    This lock file prevents several processes from
+                    attempting to modify an index at the same time.
+                </p>
+
+                <p>
+                    Note that prior to version 2.1, Lucene also used a
+                    commit lock.  This was removed in 2.1.
+		</p>
 
-                    <li>
-                        <p>
-                            When a file named "write.lock"
-                            is present, a process is currently adding documents to an index, or
-                            removing files from that index.  This lock file prevents several
-                            processes from attempting to modify an index at the same time.
-                        </p>
-                    </li>
-                </ul>
             </subsection>
 
             <subsection name="Deletable File">
 
                 <p>
-                    A file named "deletable"
-                    contains the names of files that are no longer used by the index, but
-                    which could not be deleted.  This is only used on Win32, where a
-                    file may not be deleted while it is still open. On other platforms
-                    the file contains only null bytes.
+                    Prior to Lucene 2.1 there was a file "deletable"
+                    that contained details about files that need to be
+                    deleted.  As of 2.1, a writer dynamically computes
+                    the files that are deletable, instead, so no file
+                    is written.
                 </p>
 
-                <p>
-                    Deletable    --&gt; DeletableCount,
-                    &lt;DelableName&gt;<sup>DeletableCount</sup>
-                </p>
-
-                <p>DeletableCount    --&gt; UInt32
-                </p>
-                <p>DeletableName    --&gt;
-                    String
-                </p>
             </subsection>
 
             <subsection name="Compound Files">



Mime
View raw message