lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject [5/5] lucene-solr git commit: fix a few tests; add some nocommits
Date Wed, 03 Feb 2016 09:23:18 GMT
fix a few tests; add some nocommits


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4a2bf2b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4a2bf2b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4a2bf2b

Branch: refs/heads/lucene-6835
Commit: b4a2bf2b41952be6ec3027943425d15365bcd9a0
Parents: 84f4458
Author: Mike McCandless <mikemccand@apache.org>
Authored: Tue Feb 2 18:07:06 2016 -0500
Committer: Mike McCandless <mikemccand@apache.org>
Committed: Tue Feb 2 18:07:06 2016 -0500

----------------------------------------------------------------------
 .../java/org/apache/lucene/store/Directory.java |  1 +
 .../org/apache/lucene/store/FSDirectory.java    | 55 ++++++++++++++------
 .../lucene/index/TestDirectoryReader.java       |  6 ++-
 .../org/apache/lucene/util/TestIOUtils.java     |  3 ++
 .../apache/lucene/mockfile/VirusCheckingFS.java |  4 ++
 5 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/java/org/apache/lucene/store/Directory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index b9e5ad4..bb12c92 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -47,6 +47,7 @@ public abstract class Directory implements Closeable {
    * 
    * @throws IOException in case of IO error
    */
+  // nocommit should this sort?
   public abstract String[] listAll() throws IOException;
 
   /** Removes the specified files from the directory.  If an exception is thrown, behavior
is undefined

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index 0e1d4e9..9eb78b4 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -229,22 +229,20 @@ public abstract class FSDirectory extends BaseDirectory {
     return listAll(directory, pendingDeletes);
   }
 
-  /** Returns the length in bytes of a file in the directory. */
   @Override
   public long fileLength(String name) throws IOException {
     ensureOpen();
     return Files.size(directory.resolve(name));
   }
 
-  /** Removes an existing file in the directory. */
   @Override
   public void deleteFiles(Collection<String> names) throws IOException {
     ensureOpen();
+    // nocommit isn't it an error if they were already pending delete?
     pendingDeletes.addAll(names);
     deletePendingFiles();
   }
 
-  /** Creates an IndexOutput for the file with the given name. */
   @Override
   public IndexOutput createOutput(String name, IOContext context) throws IOException {
     ensureOpen();
@@ -277,7 +275,7 @@ public abstract class FSDirectory extends BaseDirectory {
   protected void ensureCanRead(String name) throws IOException {
     deletePendingFiles();
     if (pendingDeletes.contains(name)) {
-      throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be
overwritten");
+      throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be
opened for read");
     }
   }
 
@@ -299,7 +297,6 @@ public abstract class FSDirectory extends BaseDirectory {
     IOUtils.fsync(directory, true);
   }
 
-  /** Closes the store to future operations. */
   @Override
   public synchronized void close() throws IOException {
     isOpen = false;
@@ -312,7 +309,6 @@ public abstract class FSDirectory extends BaseDirectory {
     return directory;
   }
 
-  /** For debug output. */
   @Override
   public String toString() {
     return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
@@ -324,14 +320,13 @@ public abstract class FSDirectory extends BaseDirectory {
   }
 
   /** Returns true if the file was successfully removed. */
-  private boolean deleteFile(String name) throws IOException {  
+  private synchronized boolean deleteFile(String name) throws IOException {  
+    pendingDeletes.remove(name);
     try {
       Files.delete(directory.resolve(name));
-      pendingDeletes.remove(name);
       return true;
     } catch (NoSuchFileException | FileNotFoundException e) {
       // We were asked to delete a non-existent file:
-      pendingDeletes.remove(name);
       throw e;
     } catch (IOException ioe) {
       // On windows, a file delete can fail because there's still an open
@@ -358,21 +353,36 @@ public abstract class FSDirectory extends BaseDirectory {
   }
 
   /** Try to delete any pending files that we had previously tried to delete but failed
-   *  because we are on Windows and the files were still
-   *  held open. */
-  public void deletePendingFiles() throws IOException {
+   *  because we are on Windows and the files were still held open. */
+  public synchronized void deletePendingFiles() throws IOException {
     // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they
are closed?
 
     // Clone the set because it will change as we iterate:
     List<String> toDelete = new ArrayList<>(pendingDeletes);
+    System.out.println("del pending: " + pendingDeletes);
 
     // First pass: delete any segments_N files.  We do these first to be certain stale commit
points are removed
     // before we remove any files they reference.  If any delete of segments_N fails, we
leave all other files
     // undeleted so index is never in a corrupt state:
+    Throwable firstException = null;
     for (String fileName : toDelete) {
       if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
-        if (deleteFile(fileName) == false) {
-          return;
+        try {
+          if (deleteFile(fileName) == false) {
+            // nocommit
+            System.out.println("  false on " + fileName + "; skipping the rest");
+            return;
+          }
+        } catch (Throwable t) {
+          if (firstException == null) {
+            firstException = t;
+          } else {
+            firstException.addSuppressed(t);
+          }
+          // nocommit
+          System.out.println("  fail on " + fileName + ":");
+          t.printStackTrace(System.out);
+          throw t;
         }
       }
     }
@@ -381,9 +391,24 @@ public abstract class FSDirectory extends BaseDirectory {
     // leave a corrupt commit in the index even in the presense of virus checkers:
     for(String fileName : toDelete) {
       if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
-        deleteFile(fileName);
+        try {
+          deleteFile(fileName);
+        } catch (Throwable t) {
+          if (firstException == null) {
+            firstException = t;
+          } else {
+            firstException.addSuppressed(t);
+          }
+          // nocommit
+          System.out.println("  fail on " + fileName + ":");
+          t.printStackTrace(System.out);
+          throw t;
+        }
       }
     }
+
+    // Does nothing if firstException is null:
+    IOUtils.reThrow(firstException);
   }
 
   final class FSIndexOutput extends OutputStreamIndexOutput {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 2c4f392..2213033 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -472,7 +472,10 @@ void assertTermDocsCount(String msg,
   public void testOpenReaderAfterDelete() throws IOException {
     Path dirFile = createTempDir("deletetest");
     Directory dir = newFSDirectory(dirFile);
-    assumeFalse("test deletes files directly", TestUtil.hasVirusChecker(dir));
+    if (TestUtil.hasVirusChecker(dir)) {
+      dir.close();
+      assumeTrue("test deletes files directly", false);
+    }
     if (dir instanceof BaseDirectoryWrapper) {
       ((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we will hit NoSuchFileException
in MDW since we nuked it!
     }
@@ -1055,7 +1058,6 @@ void assertTermDocsCount(String msg,
 
   public void testIndexExistsOnNonExistentDirectory() throws Exception {
     Path tempDir = createTempDir("testIndexExistsOnNonExistentDirectory");
-    IOUtils.rm(tempDir);
     Directory dir = newFSDirectory(tempDir);
     assertFalse(DirectoryReader.indexExists(dir));
     dir.close();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
index 7a112cd..726f4f8 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
@@ -38,8 +38,11 @@ import java.util.UUID;
 import org.apache.lucene.mockfile.FilterFileSystem;
 import org.apache.lucene.mockfile.FilterFileSystemProvider;
 import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.mockfile.VirusCheckingFS;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 
 /** Simple test methods for IOUtils */
+@SuppressFileSystems("VirusCheckingFS")
 public class TestIOUtils extends LuceneTestCase {
   
   public void testDeleteFileIgnoringExceptions() throws Exception {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
index 801a688..b30ce94 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
@@ -25,6 +25,7 @@ import java.nio.file.Path;
 import java.util.Random;
 
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.util.LuceneTestCase;
 
 /** 
  * Acts like Windows, where random programs may open the files you just wrote in an unfriendly
@@ -56,6 +57,9 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
         && Files.exists(path) // important that we NOT delay a NoSuchFileException
until later
         && path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false
// life is particularly difficult if the virus checker hits our lock file
         && random.nextInt(5) == 1) {
+      if (true || LuceneTestCase.VERBOSE) {
+        System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
+      }
       throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file
\"" + path + "\"");
     }
     super.delete(path);


Mime
View raw message