lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [29/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests: Removed \core directory and put its contents in root directory
Date Sun, 26 Feb 2017 23:37:17 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexWriterExceptions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexWriterExceptions.cs b/src/Lucene.Net.Tests/Index/TestIndexWriterExceptions.cs
new file mode 100644
index 0000000..5d45b7d
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexWriterExceptions.cs
@@ -0,0 +1,2584 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Randomized.Generators;
+    using Lucene.Net.Support;
+    using NUnit.Framework;
+    using System.IO;
+    using Util;
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using Analyzer = Lucene.Net.Analysis.Analyzer;
+    using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper;
+    using BinaryDocValuesField = BinaryDocValuesField;
+    using IBits = Lucene.Net.Util.IBits;
+    using BytesRef = Lucene.Net.Util.BytesRef;
+    using CannedTokenStream = Lucene.Net.Analysis.CannedTokenStream;
+    using Directory = Lucene.Net.Store.Directory;
+    using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+    using Document = Documents.Document;
+    using Field = Field;
+    using FieldType = FieldType;
+    using IndexInput = Lucene.Net.Store.IndexInput;
+    using IndexOutput = Lucene.Net.Store.IndexOutput;
+    using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+    using InfoStream = Lucene.Net.Util.InfoStream;
+    using IOContext = Lucene.Net.Store.IOContext;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+    using MockTokenizer = Lucene.Net.Analysis.MockTokenizer;
+    using NumericDocValuesField = NumericDocValuesField;
+    using PhraseQuery = Lucene.Net.Search.PhraseQuery;
+    using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+    using SortedDocValuesField = SortedDocValuesField;
+    using SortedSetDocValuesField = SortedSetDocValuesField;
+    using StringField = StringField;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+    using TextField = TextField;
+    using Token = Lucene.Net.Analysis.Token;
+    using TokenFilter = Lucene.Net.Analysis.TokenFilter;
+    using TokenStream = Lucene.Net.Analysis.TokenStream;
+
+    [TestFixture]
+    public class TestIndexWriterExceptions : LuceneTestCase
+    {
+        private class DocCopyIterator : IEnumerable<Document>
+        {
+            internal readonly Document Doc;
+            internal readonly int Count;
+
+            /* private field types */
+            /* private field types */
+
+            internal static readonly FieldType Custom1 = new FieldType(TextField.TYPE_NOT_STORED);
+            internal static readonly FieldType Custom2 = new FieldType();
+            internal static readonly FieldType Custom3 = new FieldType();
+            internal static readonly FieldType Custom4 = new FieldType(StringField.TYPE_NOT_STORED);
+            internal static readonly FieldType Custom5 = new FieldType(TextField.TYPE_STORED);
+
+            static DocCopyIterator()
+            {
+                Custom1.StoreTermVectors = true;
+                Custom1.StoreTermVectorPositions = true;
+                Custom1.StoreTermVectorOffsets = true;
+
+                Custom2.IsStored = true;
+                Custom2.IsIndexed = true;
+
+                Custom3.IsStored = true;
+
+                Custom4.StoreTermVectors = true;
+                Custom4.StoreTermVectorPositions = true;
+                Custom4.StoreTermVectorOffsets = true;
+
+                Custom5.StoreTermVectors = true;
+                Custom5.StoreTermVectorPositions = true;
+                Custom5.StoreTermVectorOffsets = true;
+            }
+
+            public DocCopyIterator(Document doc, int count)
+            {
+                this.Count = count;
+                this.Doc = doc;
+            }
+
+            public virtual IEnumerator<Document> GetEnumerator()
+            {
+                return new IteratorAnonymousInnerClassHelper(this);
+            }
+
+            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+            {
+                return GetEnumerator();
+            }
+
+            private class IteratorAnonymousInnerClassHelper : IEnumerator<Document>
+            {
+                private readonly DocCopyIterator OuterInstance;
+
+                public IteratorAnonymousInnerClassHelper(DocCopyIterator outerInstance)
+                {
+                    this.OuterInstance = outerInstance;
+                }
+
+                internal int upto;
+                private Document current;
+
+                public bool MoveNext()
+                {
+                    if (upto >= OuterInstance.Count)
+                    {
+                        return false;
+                    }
+
+                    upto++;
+                    current = OuterInstance.Doc;
+                    return true;
+                }
+
+                public Document Current
+                {
+                    get { return current; }
+                }
+
+                object System.Collections.IEnumerator.Current
+                {
+                    get { return Current; }
+                }
+
+                public void Reset()
+                {
+                    throw new NotImplementedException();
+                }
+
+                public void Dispose()
+                {
+                }
+            }
+        }
+
+        private class IndexerThread : ThreadClass
+        {
+            private DateTime unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            internal IndexWriter Writer;
+
+            internal readonly Random r = new Random(Random().Next());
+            internal volatile Exception Failure = null;
+
+            public IndexerThread(TestIndexWriterExceptions outerInstance, int i, IndexWriter writer)
+            {
+                this.OuterInstance = outerInstance;
+                Name = "Indexer " + i;
+                this.Writer = writer;
+            }
+
+            public override void Run()
+            {
+                Document doc = new Document();
+
+                doc.Add(OuterInstance.NewTextField(r, "content1", "aaa bbb ccc ddd", Field.Store.YES));
+                doc.Add(OuterInstance.NewField(r, "content6", "aaa bbb ccc ddd", DocCopyIterator.Custom1));
+                doc.Add(OuterInstance.NewField(r, "content2", "aaa bbb ccc ddd", DocCopyIterator.Custom2));
+                doc.Add(OuterInstance.NewField(r, "content3", "aaa bbb ccc ddd", DocCopyIterator.Custom3));
+
+                doc.Add(OuterInstance.NewTextField(r, "content4", "aaa bbb ccc ddd", Field.Store.NO));
+                doc.Add(OuterInstance.NewStringField(r, "content5", "aaa bbb ccc ddd", Field.Store.NO));
+                if (DefaultCodecSupportsDocValues())
+                {
+                    doc.Add(new NumericDocValuesField("numericdv", 5));
+                    doc.Add(new BinaryDocValuesField("binarydv", new BytesRef("hello")));
+                    doc.Add(new SortedDocValuesField("sorteddv", new BytesRef("world")));
+                }
+                if (DefaultCodecSupportsSortedSet())
+                {
+                    doc.Add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("hellllo")));
+                    doc.Add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("again")));
+                }
+
+                doc.Add(OuterInstance.NewField(r, "content7", "aaa bbb ccc ddd", DocCopyIterator.Custom4));
+
+                Field idField = OuterInstance.NewField(r, "id", "", DocCopyIterator.Custom2);
+                doc.Add(idField);
+
+                long stopTime = ((long)(DateTime.UtcNow - unixEpoch).TotalMilliseconds) + 500;
+
+                do
+                {
+                    if (VERBOSE)
+                    {
+                        Console.WriteLine(Thread.CurrentThread.Name + ": TEST: IndexerThread: cycle");
+                    }
+                    OuterInstance.DoFail.Value = (this.Instance);
+                    string id = "" + r.Next(50);
+                    idField.SetStringValue(id);
+                    Term idTerm = new Term("id", id);
+                    try
+                    {
+                        if (r.NextBoolean())
+                        {
+                            Writer.UpdateDocuments(idTerm, new DocCopyIterator(doc, TestUtil.NextInt(r, 1, 20)));
+                        }
+                        else
+                        {
+                            Writer.UpdateDocument(idTerm, doc);
+                        }
+                    }
+                    catch (TestPoint1Exception re)
+                    {
+                        if (VERBOSE)
+                        {
+                            Console.WriteLine(Thread.CurrentThread.Name + ": EXC: ");
+                            Console.WriteLine(re.StackTrace);
+                        }
+                        try
+                        {
+                            TestUtil.CheckIndex(Writer.Directory);
+                        }
+                        catch (IOException ioe)
+                        {
+                            Console.WriteLine(Thread.CurrentThread.Name + ": unexpected exception1");
+                            Console.WriteLine(ioe.StackTrace);
+                            Failure = ioe;
+                            break;
+                        }
+                    }
+                    catch (Exception t)
+                    {
+                        Console.WriteLine(Thread.CurrentThread.Name + ": unexpected exception2");
+                        Console.WriteLine(t.StackTrace);
+                        Failure = t;
+                        break;
+                    }
+
+                    OuterInstance.DoFail.Value = (null);
+
+                    // After a possible exception (above) I should be able
+                    // to add a new document without hitting an
+                    // exception:
+                    try
+                    {
+                        Writer.UpdateDocument(idTerm, doc);
+                    }
+                    catch (Exception t)
+                    {
+                        Console.WriteLine(Thread.CurrentThread.Name + ": unexpected exception3");
+                        Console.WriteLine(t.StackTrace);
+                        Failure = t;
+                        break;
+                    }
+                } while (((long)(DateTime.UtcNow - unixEpoch).TotalMilliseconds) < stopTime);
+            }
+        }
+
+        internal ThreadLocal<Thread> DoFail = new ThreadLocal<Thread>();
+
+        private class TestPoint1 : RandomIndexWriter.TestPoint
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public TestPoint1(TestIndexWriterExceptions outerInstance)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            internal Random r = new Random(Random().Next());
+
+            public void Apply(string name)
+            {
+                if (OuterInstance.DoFail.Value != null && !name.Equals("startDoFlush") && r.Next(40) == 17)
+                {
+                    if (VERBOSE)
+                    {
+                        Console.WriteLine(Thread.CurrentThread.Name + ": NOW FAIL: " + name);
+                        Console.WriteLine((new Exception()).StackTrace);
+                    }
+                    throw new TestPoint1Exception(Thread.CurrentThread.Name + ": intentionally failing at " + name);
+                }
+            }
+        }
+
+        private class TestPoint1Exception : Exception
+        {
+            public TestPoint1Exception(string message) : base(message)
+            {
+            }
+        }
+
+        [Test]
+        public virtual void TestRandomExceptions([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+        {
+            if (VERBOSE)
+            {
+                Console.WriteLine("\nTEST: start testRandomExceptions");
+            }
+            Directory dir = NewDirectory();
+
+            MockAnalyzer analyzer = new MockAnalyzer(Random());
+            analyzer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+
+
+            var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
+                            .SetRAMBufferSizeMB(0.1)
+                            .SetMergeScheduler(scheduler);
+
+            scheduler.SetSuppressExceptions();
+
+            IndexWriter writer = RandomIndexWriter.MockIndexWriter(dir, config , new TestPoint1(this));
+            //writer.SetMaxBufferedDocs(10);
+            if (VERBOSE)
+            {
+                Console.WriteLine("TEST: initial commit");
+            }
+            writer.Commit();
+
+            IndexerThread thread = new IndexerThread(this, 0, writer);
+            thread.Run();
+            if (thread.Failure != null)
+            {
+                Console.WriteLine(thread.Failure.StackTrace);
+                Assert.Fail("thread " + thread.Name + ": hit unexpected failure");
+            }
+
+            if (VERBOSE)
+            {
+                Console.WriteLine("TEST: commit after thread start");
+            }
+            writer.Commit();
+
+            try
+            {
+                writer.Dispose();
+            }
+            catch (Exception t)
+            {
+                Console.WriteLine("exception during close:");
+                Console.WriteLine(t.StackTrace);
+                writer.Rollback();
+            }
+
+            // Confirm that when doc hits exception partway through tokenization, it's deleted:
+            IndexReader r2 = DirectoryReader.Open(dir);
+            int count = r2.DocFreq(new Term("content4", "aaa"));
+            int count2 = r2.DocFreq(new Term("content4", "ddd"));
+            Assert.AreEqual(count, count2);
+            r2.Dispose();
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestRandomExceptionsThreads([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+        {
+            Directory dir = NewDirectory();
+            MockAnalyzer analyzer = new MockAnalyzer(Random());
+            analyzer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+
+            var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
+                            .SetRAMBufferSizeMB(0.2)
+                            .SetMergeScheduler(scheduler);
+
+            IndexWriter writer = RandomIndexWriter.MockIndexWriter(dir, config, new TestPoint1(this));
+            scheduler.SetSuppressExceptions();
+
+            //writer.SetMaxBufferedDocs(10);
+            writer.Commit();
+
+            const int NUM_THREADS = 4;
+
+            IndexerThread[] threads = new IndexerThread[NUM_THREADS];
+            for (int i = 0; i < NUM_THREADS; i++)
+            {
+                threads[i] = new IndexerThread(this, i, writer);
+                threads[i].Start();
+            }
+
+            for (int i = 0; i < NUM_THREADS; i++)
+            {
+                threads[i].Join();
+            }
+
+            for (int i = 0; i < NUM_THREADS; i++)
+            {
+                if (threads[i].Failure != null)
+                {
+                    Assert.Fail("thread " + threads[i].Name + ": hit unexpected failure");
+                }
+            }
+
+            writer.Commit();
+
+            try
+            {
+                writer.Dispose();
+            }
+            catch (Exception t)
+            {
+                Console.WriteLine("exception during close:");
+                Console.WriteLine(t.StackTrace);
+                writer.Rollback();
+            }
+
+            // Confirm that when doc hits exception partway through tokenization, it's deleted:
+            IndexReader r2 = DirectoryReader.Open(dir);
+            int count = r2.DocFreq(new Term("content4", "aaa"));
+            int count2 = r2.DocFreq(new Term("content4", "ddd"));
+            Assert.AreEqual(count, count2);
+            r2.Dispose();
+
+            dir.Dispose();
+        }
+
+        // LUCENE-1198
+        private sealed class TestPoint2 : RandomIndexWriter.TestPoint
+        {
+            internal bool DoFail;
+
+            public void Apply(string name)
+            {
+                if (DoFail && name.Equals("DocumentsWriterPerThread addDocument start"))
+                {
+                    throw new Exception("intentionally failing");
+                }
+            }
+        }
+
+        private static string CRASH_FAIL_MESSAGE = "I'm experiencing problems";
+
+        private class CrashingFilter : TokenFilter
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            internal string FieldName;
+            internal int Count;
+
+            public CrashingFilter(TestIndexWriterExceptions outerInstance, string fieldName, TokenStream input)
+                : base(input)
+            {
+                this.OuterInstance = outerInstance;
+                this.FieldName = fieldName;
+            }
+
+            public sealed override bool IncrementToken()
+            {
+                if (this.FieldName.Equals("crash") && Count++ >= 4)
+                {
+                    throw new IOException(CRASH_FAIL_MESSAGE);
+                }
+                return m_input.IncrementToken();
+            }
+
+            public override void Reset()
+            {
+                base.Reset();
+                Count = 0;
+            }
+        }
+
+        [Test]
+        public virtual void TestExceptionDocumentsWriterInit()
+        {
+            Directory dir = NewDirectory();
+            TestPoint2 testPoint = new TestPoint2();
+            IndexWriter w = RandomIndexWriter.MockIndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())), testPoint);
+            Document doc = new Document();
+            doc.Add(NewTextField("field", "a field", Field.Store.YES));
+            w.AddDocument(doc);
+            testPoint.DoFail = true;
+            try
+            {
+                w.AddDocument(doc);
+                Assert.Fail("did not hit exception");
+            }
+#pragma warning disable 168
+            catch (Exception re)
+#pragma warning restore 168
+            {
+                // expected
+            }
+            w.Dispose();
+            dir.Dispose();
+        }
+
+        // LUCENE-1208
+        [Test]
+        public virtual void TestExceptionJustBeforeFlush()
+        {
+            Directory dir = NewDirectory();
+            IndexWriter w = RandomIndexWriter.MockIndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2), new TestPoint1(this));
+            Document doc = new Document();
+            doc.Add(NewTextField("field", "a field", Field.Store.YES));
+            w.AddDocument(doc);
+
+            Analyzer analyzer = new TEJBFAnalyzerAnonymousInnerClassHelper(this, Analyzer.PER_FIELD_REUSE_STRATEGY);
+
+            Document crashDoc = new Document();
+            crashDoc.Add(NewTextField("crash", "do it on token 4", Field.Store.YES));
+            try
+            {
+                w.AddDocument(crashDoc, analyzer);
+                Assert.Fail("did not hit expected exception");
+            }
+#pragma warning disable 168
+            catch (IOException ioe)
+#pragma warning restore 168
+            {
+                // expected
+            }
+            w.AddDocument(doc);
+            w.Dispose();
+            dir.Dispose();
+        }
+
+        private class TEJBFAnalyzerAnonymousInnerClassHelper : Analyzer
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public TEJBFAnalyzerAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance, Analyzer.ReuseStrategy PER_FIELD_REUSE_STRATEGY)
+                : base(PER_FIELD_REUSE_STRATEGY)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+            {
+                MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+                tokenizer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+                return new TokenStreamComponents(tokenizer, new CrashingFilter(OuterInstance, fieldName, tokenizer));
+            }
+        }
+
+        private sealed class TestPoint3 : RandomIndexWriter.TestPoint
+        {
+            internal bool DoFail;
+            internal bool Failed;
+
+            public void Apply(string name)
+            {
+                if (DoFail && name.Equals("startMergeInit"))
+                {
+                    Failed = true;
+                    throw new Exception("intentionally failing");
+                }
+            }
+        }
+
+        // LUCENE-1210
+        [Test]
+        public virtual void TestExceptionOnMergeInit([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+        {
+            Directory dir = NewDirectory();
+            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy());
+
+            scheduler.SetSuppressExceptions();
+            conf.SetMergeScheduler(scheduler);
+            ((LogMergePolicy)conf.MergePolicy).MergeFactor = 2;
+            TestPoint3 testPoint = new TestPoint3();
+            IndexWriter w = RandomIndexWriter.MockIndexWriter(dir, conf, testPoint);
+            testPoint.DoFail = true;
+            Document doc = new Document();
+            doc.Add(NewTextField("field", "a field", Field.Store.YES));
+            for (int i = 0; i < 10; i++)
+            {
+                try
+                {
+                    w.AddDocument(doc);
+                }
+                catch (Exception)
+                {
+                    break;
+                }
+            }
+
+            ((IConcurrentMergeScheduler)w.Config.MergeScheduler).Sync();
+            Assert.IsTrue(testPoint.Failed);
+            w.Dispose();
+            dir.Dispose();
+        }
+
+        // LUCENE-1072
+        [Test]
+        public virtual void TestExceptionFromTokenStream()
+        {
+            Directory dir = NewDirectory();
+            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new TEFTSAnalyzerAnonymousInnerClassHelper(this));
+            conf.SetMaxBufferedDocs(Math.Max(3, conf.MaxBufferedDocs));
+
+            IndexWriter writer = new IndexWriter(dir, conf);
+
+            Document doc = new Document();
+            string contents = "aa bb cc dd ee ff gg hh ii jj kk";
+            doc.Add(NewTextField("content", contents, Field.Store.NO));
+            try
+            {
+                writer.AddDocument(doc);
+                Assert.Fail("did not hit expected exception");
+            }
+#pragma warning disable 168
+            catch (Exception e)
+#pragma warning restore 168
+            {
+            }
+
+            // Make sure we can add another normal document
+            doc = new Document();
+            doc.Add(NewTextField("content", "aa bb cc dd", Field.Store.NO));
+            writer.AddDocument(doc);
+
+            // Make sure we can add another normal document
+            doc = new Document();
+            doc.Add(NewTextField("content", "aa bb cc dd", Field.Store.NO));
+            writer.AddDocument(doc);
+
+            writer.Dispose();
+            IndexReader reader = DirectoryReader.Open(dir);
+            Term t = new Term("content", "aa");
+            Assert.AreEqual(3, reader.DocFreq(t));
+
+            // Make sure the doc that hit the exception was marked
+            // as deleted:
+            DocsEnum tdocs = TestUtil.Docs(Random(), reader, t.Field, new BytesRef(t.Text()), MultiFields.GetLiveDocs(reader), null, 0);
+
+            int count = 0;
+            while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+            {
+                count++;
+            }
+            Assert.AreEqual(2, count);
+
+            Assert.AreEqual(reader.DocFreq(new Term("content", "gg")), 0);
+            reader.Dispose();
+            dir.Dispose();
+        }
+
+        private class TEFTSAnalyzerAnonymousInnerClassHelper : Analyzer
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public TEFTSAnalyzerAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+            {
+                MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+                tokenizer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+                return new TokenStreamComponents(tokenizer, new TokenFilterAnonymousInnerClassHelper(this, tokenizer));
+            }
+
+            private class TokenFilterAnonymousInnerClassHelper : TokenFilter
+            {
+                private readonly TEFTSAnalyzerAnonymousInnerClassHelper OuterInstance;
+
+                public TokenFilterAnonymousInnerClassHelper(TEFTSAnalyzerAnonymousInnerClassHelper outerInstance, MockTokenizer tokenizer)
+                    : base(tokenizer)
+                {
+                    this.OuterInstance = outerInstance;
+                    count = 0;
+                }
+
+                private int count;
+
+                public sealed override bool IncrementToken()
+                {
+                    if (count++ == 5)
+                    {
+                        throw new IOException();
+                    }
+                    return m_input.IncrementToken();
+                }
+
+                public override void Reset()
+                {
+                    base.Reset();
+                    this.count = 0;
+                }
+            }
+        }
+
+        private class FailOnlyOnFlush : MockDirectoryWrapper.Failure
+        {
+            new internal bool DoFail = false;
+            internal int Count;
+
+            public override void SetDoFail()
+            {
+                this.DoFail = true;
+            }
+
+            public override void ClearDoFail()
+            {
+                this.DoFail = false;
+            }
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                if (DoFail)
+                {
+                    bool sawAppend = StackTraceHelper.DoesStackTraceContainMethod(typeof(FreqProxTermsWriterPerField).Name, "Flush");
+                    bool sawFlush = StackTraceHelper.DoesStackTraceContainMethod("Flush");
+
+                    if (sawAppend && sawFlush && Count++ >= 30)
+                    {
+                        DoFail = false;
+                        throw new IOException("now failing during flush");
+                    }
+                }
+            }
+        }
+
+        // LUCENE-1072: make sure an errant exception on flushing
+        // one segment only takes out those docs in that one flush
+        [Test]
+        public virtual void TestDocumentsWriterAbort()
+        {
+            MockDirectoryWrapper dir = NewMockDirectory();
+            FailOnlyOnFlush failure = new FailOnlyOnFlush();
+            failure.SetDoFail();
+            dir.FailOn(failure);
+
+            IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
+            Document doc = new Document();
+            string contents = "aa bb cc dd ee ff gg hh ii jj kk";
+            doc.Add(NewTextField("content", contents, Field.Store.NO));
+            bool hitError = false;
+            for (int i = 0; i < 200; i++)
+            {
+                try
+                {
+                    writer.AddDocument(doc);
+                }
+#pragma warning disable 168
+                catch (IOException ioe)
+#pragma warning restore 168
+                {
+                    // only one flush should fail:
+                    Assert.IsFalse(hitError);
+                    hitError = true;
+                }
+            }
+            Assert.IsTrue(hitError);
+            writer.Dispose();
+            IndexReader reader = DirectoryReader.Open(dir);
+            Assert.AreEqual(198, reader.DocFreq(new Term("content", "aa")));
+            reader.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestDocumentsWriterExceptions()
+        {
+            Analyzer analyzer = new TDWEAnalyzerAnonymousInnerClassHelper(this, Analyzer.PER_FIELD_REUSE_STRATEGY);
+
+            for (int i = 0; i < 2; i++)
+            {
+                if (VERBOSE)
+                {
+                    Console.WriteLine("TEST: cycle i=" + i);
+                }
+                Directory dir = NewDirectory();
+                IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMergePolicy(NewLogMergePolicy()));
+
+                // don't allow a sudden merge to clean up the deleted
+                // doc below:
+                LogMergePolicy lmp = (LogMergePolicy)writer.Config.MergePolicy;
+                lmp.MergeFactor = Math.Max(lmp.MergeFactor, 5);
+
+                Document doc = new Document();
+                doc.Add(NewField("contents", "here are some contents", DocCopyIterator.Custom5));
+                writer.AddDocument(doc);
+                writer.AddDocument(doc);
+                doc.Add(NewField("crash", "this should crash after 4 terms", DocCopyIterator.Custom5));
+                doc.Add(NewField("other", "this will not get indexed", DocCopyIterator.Custom5));
+                try
+                {
+                    writer.AddDocument(doc);
+                    Assert.Fail("did not hit expected exception");
+                }
+                catch (IOException ioe)
+                {
+                    if (VERBOSE)
+                    {
+                        Console.WriteLine("TEST: hit expected exception");
+                        Console.WriteLine(ioe.StackTrace);
+                    }
+                }
+
+                if (0 == i)
+                {
+                    doc = new Document();
+                    doc.Add(NewField("contents", "here are some contents", DocCopyIterator.Custom5));
+                    writer.AddDocument(doc);
+                    writer.AddDocument(doc);
+                }
+                writer.Dispose();
+
+                if (VERBOSE)
+                {
+                    Console.WriteLine("TEST: open reader");
+                }
+                IndexReader reader = DirectoryReader.Open(dir);
+                if (i == 0)
+                {
+                    int expected = 5;
+                    Assert.AreEqual(expected, reader.DocFreq(new Term("contents", "here")));
+                    Assert.AreEqual(expected, reader.MaxDoc);
+                    int numDel = 0;
+                    IBits liveDocs = MultiFields.GetLiveDocs(reader);
+                    Assert.IsNotNull(liveDocs);
+                    for (int j = 0; j < reader.MaxDoc; j++)
+                    {
+                        if (!liveDocs.Get(j))
+                        {
+                            numDel++;
+                        }
+                        else
+                        {
+                            reader.Document(j);
+                            reader.GetTermVectors(j);
+                        }
+                    }
+                    Assert.AreEqual(1, numDel);
+                }
+                reader.Dispose();
+
+                writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10));
+                doc = new Document();
+                doc.Add(NewField("contents", "here are some contents", DocCopyIterator.Custom5));
+                for (int j = 0; j < 17; j++)
+                {
+                    writer.AddDocument(doc);
+                }
+                writer.ForceMerge(1);
+                writer.Dispose();
+
+                reader = DirectoryReader.Open(dir);
+                int expected_ = 19 + (1 - i) * 2;
+                Assert.AreEqual(expected_, reader.DocFreq(new Term("contents", "here")));
+                Assert.AreEqual(expected_, reader.MaxDoc);
+                int numDel_ = 0;
+                Assert.IsNull(MultiFields.GetLiveDocs(reader));
+                for (int j = 0; j < reader.MaxDoc; j++)
+                {
+                    reader.Document(j);
+                    reader.GetTermVectors(j);
+                }
+                reader.Dispose();
+                Assert.AreEqual(0, numDel_);
+
+                dir.Dispose();
+            }
+        }
+
+        private class TDWEAnalyzerAnonymousInnerClassHelper : Analyzer
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public TDWEAnalyzerAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance, Analyzer.ReuseStrategy PER_FIELD_REUSE_STRATEGY)
+                : base(PER_FIELD_REUSE_STRATEGY)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+            {
+                MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+                tokenizer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+                return new TokenStreamComponents(tokenizer, new CrashingFilter(OuterInstance, fieldName, tokenizer));
+            }
+        }
+
+        [Test]
+        public virtual void TestDocumentsWriterExceptionThreads()
+        {
+            Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper2(this, Analyzer.PER_FIELD_REUSE_STRATEGY);
+
+            const int NUM_THREAD = 3;
+            const int NUM_ITER = 100;
+
+            for (int i = 0; i < 2; i++)
+            {
+                Directory dir = NewDirectory();
+
+                {
+                    IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(-1).SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES));
+                    // don't use a merge policy here they depend on the DWPThreadPool and its max thread states etc.
+                    int finalI = i;
+
+                    ThreadClass[] threads = new ThreadClass[NUM_THREAD];
+                    for (int t = 0; t < NUM_THREAD; t++)
+                    {
+                        threads[t] = new ThreadAnonymousInnerClassHelper(this, NUM_ITER, writer, finalI, t);
+                        threads[t].Start();
+                    }
+
+                    for (int t = 0; t < NUM_THREAD; t++)
+                    {
+                        threads[t].Join();
+                    }
+
+                    writer.Dispose();
+                }
+
+                IndexReader reader = DirectoryReader.Open(dir);
+                int expected = (3 + (1 - i) * 2) * NUM_THREAD * NUM_ITER;
+                Assert.AreEqual(expected, reader.DocFreq(new Term("contents", "here")), "i=" + i);
+                Assert.AreEqual(expected, reader.MaxDoc);
+                int numDel = 0;
+                IBits liveDocs = MultiFields.GetLiveDocs(reader);
+                Assert.IsNotNull(liveDocs);
+                for (int j = 0; j < reader.MaxDoc; j++)
+                {
+                    if (!liveDocs.Get(j))
+                    {
+                        numDel++;
+                    }
+                    else
+                    {
+                        reader.Document(j);
+                        reader.GetTermVectors(j);
+                    }
+                }
+                reader.Dispose();
+
+                Assert.AreEqual(NUM_THREAD * NUM_ITER, numDel);
+
+                IndexWriter indWriter = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10));
+                Document doc = new Document();
+                doc.Add(NewField("contents", "here are some contents", DocCopyIterator.Custom5));
+                for (int j = 0; j < 17; j++)
+                {
+                    indWriter.AddDocument(doc);
+                }
+                indWriter.ForceMerge(1);
+                indWriter.Dispose();
+
+                reader = DirectoryReader.Open(dir);
+                expected += 17 - NUM_THREAD * NUM_ITER;
+                Assert.AreEqual(expected, reader.DocFreq(new Term("contents", "here")));
+                Assert.AreEqual(expected, reader.MaxDoc);
+                Assert.IsNull(MultiFields.GetLiveDocs(reader));
+                for (int j = 0; j < reader.MaxDoc; j++)
+                {
+                    reader.Document(j);
+                    reader.GetTermVectors(j);
+                }
+                reader.Dispose();
+
+                dir.Dispose();
+            }
+        }
+
+        private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public AnalyzerAnonymousInnerClassHelper2(TestIndexWriterExceptions outerInstance, Analyzer.ReuseStrategy PER_FIELD_REUSE_STRATEGY)
+                : base(PER_FIELD_REUSE_STRATEGY)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+            {
+                MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+                tokenizer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+                return new TokenStreamComponents(tokenizer, new CrashingFilter(OuterInstance, fieldName, tokenizer));
+            }
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            private int NUM_ITER;
+            private IndexWriter Writer;
+            private int FinalI;
+            private int t;
+
+            public ThreadAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance, int NUM_ITER, IndexWriter writer, int finalI, int t)
+            {
+                this.OuterInstance = outerInstance;
+                this.NUM_ITER = NUM_ITER;
+                this.Writer = writer;
+                this.FinalI = finalI;
+                this.t = t;
+            }
+
+            public override void Run()
+            {
+                try
+                {
+                    for (int iter = 0; iter < NUM_ITER; iter++)
+                    {
+                        Document doc = new Document();
+                        doc.Add(OuterInstance.NewField("contents", "here are some contents", DocCopyIterator.Custom5));
+                        Writer.AddDocument(doc);
+                        Writer.AddDocument(doc);
+                        doc.Add(OuterInstance.NewField("crash", "this should crash after 4 terms", DocCopyIterator.Custom5));
+                        doc.Add(OuterInstance.NewField("other", "this will not get indexed", DocCopyIterator.Custom5));
+                        try
+                        {
+                            Writer.AddDocument(doc);
+                            Assert.Fail("did not hit expected exception");
+                        }
+#pragma warning disable 168
+                        catch (IOException ioe)
+#pragma warning restore 168
+                        {
+                        }
+
+                        if (0 == FinalI)
+                        {
+                            doc = new Document();
+                            doc.Add(OuterInstance.NewField("contents", "here are some contents", DocCopyIterator.Custom5));
+                            Writer.AddDocument(doc);
+                            Writer.AddDocument(doc);
+                        }
+                    }
+                }
+                catch (Exception t)
+                {
+                    lock (this)
+                    {
+                        Console.WriteLine(Thread.CurrentThread.Name + ": ERROR: hit unexpected exception");
+                        Console.WriteLine(t.StackTrace);
+                    }
+                    Assert.Fail();
+                }
+            }
+        }
+
+        // Throws IOException during MockDirectoryWrapper.sync
+        private class FailOnlyInSync : MockDirectoryWrapper.Failure
+        {
+            internal bool DidFail;
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                if (DoFail)
+                {
+                    bool foundMethod =
+                        StackTraceHelper.DoesStackTraceContainMethod(typeof(MockDirectoryWrapper).Name, "Sync");
+
+                    if (DoFail && foundMethod)
+                    {
+                        DidFail = true;
+                        if (VERBOSE)
+                        {
+                            Console.WriteLine("TEST: now throw exc:");
+                            Console.WriteLine(Environment.StackTrace);
+                        }
+                        throw new IOException("now failing on purpose during sync");
+                    }
+                }
+            }
+        }
+
+        // TODO: these are also in TestIndexWriter... add a simple doc-writing method
+        // like this to LuceneTestCase?
+        private void AddDoc(IndexWriter writer)
+        {
+            Document doc = new Document();
+            doc.Add(NewTextField("content", "aaa", Field.Store.NO));
+            writer.AddDocument(doc);
+        }
+
+        // LUCENE-1044: test exception during sync
+        [Test]
+        public virtual void TestExceptionDuringSync([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+        {
+            MockDirectoryWrapper dir = NewMockDirectory();
+            FailOnlyInSync failure = new FailOnlyInSync();
+            dir.FailOn(failure);
+
+            var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
+                            .SetMaxBufferedDocs(2)
+                            .SetMergeScheduler(scheduler)
+                            .SetMergePolicy(NewLogMergePolicy(5));
+
+            IndexWriter writer = new IndexWriter(dir, config);
+            failure.SetDoFail();
+
+            for (int i = 0; i < 23; i++)
+            {
+                AddDoc(writer);
+                if ((i - 1) % 2 == 0)
+                {
+                    try
+                    {
+                        writer.Commit();
+                    }
+#pragma warning disable 168
+                    catch (IOException ioe)
+#pragma warning restore 168
+                    {
+                        // expected
+                    }
+                }
+            }
+            ((IConcurrentMergeScheduler)writer.Config.MergeScheduler).Sync();
+            Assert.IsTrue(failure.DidFail);
+            failure.ClearDoFail();
+            writer.Dispose();
+
+            IndexReader reader = DirectoryReader.Open(dir);
+            Assert.AreEqual(23, reader.NumDocs);
+            reader.Dispose();
+            dir.Dispose();
+        }
+
+        private class FailOnlyInCommit : MockDirectoryWrapper.Failure
+        {
+            internal bool FailOnCommit, FailOnDeleteFile;
+            internal readonly bool DontFailDuringGlobalFieldMap;
+            internal const string PREPARE_STAGE = "PrepareCommit";
+            internal const string FINISH_STAGE = "FinishCommit";
+            internal readonly string Stage;
+
+            public FailOnlyInCommit(bool dontFailDuringGlobalFieldMap, string stage)
+            {
+                this.DontFailDuringGlobalFieldMap = dontFailDuringGlobalFieldMap;
+                this.Stage = stage;
+            }
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                bool isCommit = StackTraceHelper.DoesStackTraceContainMethod(typeof(SegmentInfos).Name, Stage);
+                bool isDelete = StackTraceHelper.DoesStackTraceContainMethod(typeof(MockDirectoryWrapper).Name, "DeleteFile");
+                bool isInGlobalFieldMap = StackTraceHelper.DoesStackTraceContainMethod(typeof(SegmentInfos).Name, "WriteGlobalFieldMap");
+
+                if (isInGlobalFieldMap && DontFailDuringGlobalFieldMap)
+                {
+                    isCommit = false;
+                }
+                if (isCommit)
+                {
+                    if (!isDelete)
+                    {
+                        FailOnCommit = true;
+                        throw new Exception("now fail first");
+                    }
+                    else
+                    {
+                        FailOnDeleteFile = true;
+                        throw new IOException("now fail during delete");
+                    }
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestExceptionsDuringCommit()
+        {
+            FailOnlyInCommit[] failures = new FailOnlyInCommit[] { new FailOnlyInCommit(false, FailOnlyInCommit.PREPARE_STAGE), new FailOnlyInCommit(true, FailOnlyInCommit.PREPARE_STAGE), new FailOnlyInCommit(false, FailOnlyInCommit.FINISH_STAGE) };
+
+            foreach (FailOnlyInCommit failure in failures)
+            {
+                MockDirectoryWrapper dir = NewMockDirectory();
+                dir.FailOnCreateOutput = false;
+                IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+                Document doc = new Document();
+                doc.Add(NewTextField("field", "a field", Field.Store.YES));
+                w.AddDocument(doc);
+                dir.FailOn(failure);
+                try
+                {
+                    w.Dispose();
+                    Assert.Fail();
+                }
+#pragma warning disable 168
+                catch (IOException ioe)
+#pragma warning restore 168
+                {
+                    Assert.Fail("expected only RuntimeException");
+                }
+#pragma warning disable 168
+                catch (Exception re)
+#pragma warning restore 168
+                {
+                    // Expected
+                }
+                Assert.IsTrue(failure.FailOnCommit && failure.FailOnDeleteFile);
+                w.Rollback();
+                string[] files = dir.ListAll();
+                Assert.IsTrue(files.Length == 0 || Arrays.Equals(files, new string[] { IndexWriter.WRITE_LOCK_NAME }));
+                dir.Dispose();
+            }
+        }
+
+        [Test]
+        public virtual void TestForceMergeExceptions([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+        {
+            Directory startDir = NewDirectory();
+            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy());
+            ((LogMergePolicy)conf.MergePolicy).MergeFactor = 100;
+            IndexWriter w = new IndexWriter(startDir, conf);
+            for (int i = 0; i < 27; i++)
+            {
+                AddDoc(w);
+            }
+            w.Dispose();
+
+            int iter = TEST_NIGHTLY ? 200 : 10;
+            for (int i = 0; i < iter; i++)
+            {
+                if (VERBOSE)
+                {
+                    Console.WriteLine("TEST: iter " + i);
+                }
+                MockDirectoryWrapper dir = new MockDirectoryWrapper(Random(), new RAMDirectory(startDir, NewIOContext(Random())));
+                conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(scheduler);
+                scheduler.SetSuppressExceptions();
+                w = new IndexWriter(dir, conf);
+                dir.RandomIOExceptionRate = 0.5;
+                try
+                {
+                    w.ForceMerge(1);
+                }
+                catch (IOException ioe)
+                {
+                    if (ioe.InnerException == null)
+                    {
+                        Assert.Fail("forceMerge threw IOException without root cause");
+                    }
+                }
+                dir.RandomIOExceptionRate = 0;
+                w.Dispose();
+                dir.Dispose();
+            }
+            startDir.Dispose();
+        }
+
+        // LUCENE-1429
+        [Test]
+        public virtual void TestOutOfMemoryErrorCausesCloseToFail()
+        {
+            AtomicBoolean thrown = new AtomicBoolean(false);
+            Directory dir = NewDirectory();
+            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetInfoStream(new TOOMInfoStreamAnonymousInnerClassHelper(this, thrown)));
+
+            try
+            {
+                writer.Dispose();
+                Assert.Fail("OutOfMemoryError expected");
+            }
+#pragma warning disable 168
+            catch (System.OutOfMemoryException expected)
+#pragma warning restore 168
+            {
+            }
+
+            // throws IllegalStateEx w/o bug fix
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        private class TOOMInfoStreamAnonymousInnerClassHelper : InfoStream
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            private AtomicBoolean Thrown;
+
+            public TOOMInfoStreamAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance, AtomicBoolean thrown)
+            {
+                this.OuterInstance = outerInstance;
+                this.Thrown = thrown;
+            }
+
+            public override void Message(string component, string message)
+            {
+                if (message.StartsWith("now flush at close") && Thrown.CompareAndSet(false, true))
+                {
+                    throw new System.OutOfMemoryException("fake OOME at " + message);
+                }
+            }
+
+            public override bool IsEnabled(string component)
+            {
+                return true;
+            }
+
+            public override void Dispose()
+            {
+            }
+        }
+
+        // LUCENE-1347
+        private sealed class TestPoint4 : RandomIndexWriter.TestPoint
+        {
+            internal bool DoFail;
+
+            public void Apply(string name)
+            {
+                if (DoFail && name.Equals("rollback before checkpoint"))
+                {
+                    throw new Exception("intentionally failing");
+                }
+            }
+        }
+
+        // LUCENE-1347
+        [Test]
+        public virtual void TestRollbackExceptionHang()
+        {
+            Directory dir = NewDirectory();
+            TestPoint4 testPoint = new TestPoint4();
+            IndexWriter w = RandomIndexWriter.MockIndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())), testPoint);
+
+            AddDoc(w);
+            testPoint.DoFail = true;
+            try
+            {
+                w.Rollback();
+                Assert.Fail("did not hit intentional RuntimeException");
+            }
+#pragma warning disable 168
+            catch (Exception re)
+#pragma warning restore 168
+            {
+                // expected
+            }
+
+            testPoint.DoFail = false;
+            w.Rollback();
+            dir.Dispose();
+        }
+
+        // LUCENE-1044: Simulate checksum error in segments_N
+        [Test]
+        public virtual void TestSegmentsChecksumError()
+        {
+            Directory dir = NewDirectory();
+
+            IndexWriter writer = null;
+
+            writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+            // add 100 documents
+            for (int i = 0; i < 100; i++)
+            {
+                AddDoc(writer);
+            }
+
+            // close
+            writer.Dispose();
+
+            long gen = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.IsTrue(gen > 0, "segment generation should be > 0 but got " + gen);
+
+            string segmentsFileName = SegmentInfos.GetLastCommitSegmentsFileName(dir);
+            IndexInput @in = dir.OpenInput(segmentsFileName, NewIOContext(Random()));
+            IndexOutput @out = dir.CreateOutput(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen), NewIOContext(Random()));
+            @out.CopyBytes(@in, @in.Length - 1);
+            byte b = @in.ReadByte();
+            @out.WriteByte((byte)(sbyte)(1 + b));
+            @out.Dispose();
+            @in.Dispose();
+
+            IndexReader reader = null;
+            try
+            {
+                reader = DirectoryReader.Open(dir);
+            }
+            catch (IOException e)
+            {
+                Console.WriteLine(e.StackTrace);
+                Assert.Fail("segmentInfos failed to retry fallback to correct segments_N file");
+            }
+            reader.Dispose();
+
+            // should remove the corrumpted segments_N
+            (new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
+            dir.Dispose();
+        }
+
+        // Simulate a corrupt index by removing last byte of
+        // latest segments file and make sure we get an
+        // IOException trying to open the index:
+        [Test]
+        public virtual void TestSimulatedCorruptIndex1()
+        {
+            BaseDirectoryWrapper dir = NewDirectory();
+            dir.CheckIndexOnClose = false; // we are corrupting it!
+
+            IndexWriter writer = null;
+
+            writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+            // add 100 documents
+            for (int i = 0; i < 100; i++)
+            {
+                AddDoc(writer);
+            }
+
+            // close
+            writer.Dispose();
+
+            long gen = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.IsTrue(gen > 0, "segment generation should be > 0 but got " + gen);
+
+            string fileNameIn = SegmentInfos.GetLastCommitSegmentsFileName(dir);
+            string fileNameOut = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen);
+            IndexInput @in = dir.OpenInput(fileNameIn, NewIOContext(Random()));
+            IndexOutput @out = dir.CreateOutput(fileNameOut, NewIOContext(Random()));
+            long length = @in.Length;
+            for (int i = 0; i < length - 1; i++)
+            {
+                @out.WriteByte(@in.ReadByte());
+            }
+            @in.Dispose();
+            @out.Dispose();
+            dir.DeleteFile(fileNameIn);
+
+            IndexReader reader = null;
+            try
+            {
+                reader = DirectoryReader.Open(dir);
+                Assert.Fail("reader did not hit IOException on opening a corrupt index");
+            }
+#pragma warning disable 168
+            catch (Exception e)
+#pragma warning restore 168
+            {
+            }
+            if (reader != null)
+            {
+                reader.Dispose();
+            }
+            dir.Dispose();
+        }
+
+        // Simulate a corrupt index by removing one of the cfs
+        // files and make sure we get an IOException trying to
+        // open the index:
+        [Test]
+        public virtual void TestSimulatedCorruptIndex2()
+        {
+            BaseDirectoryWrapper dir = NewDirectory();
+            dir.CheckIndexOnClose = false; // we are corrupting it!
+            IndexWriter writer = null;
+
+            writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy(true)).SetUseCompoundFile(true));
+            MergePolicy lmp = writer.Config.MergePolicy;
+            // Force creation of CFS:
+            lmp.NoCFSRatio = 1.0;
+            lmp.MaxCFSSegmentSizeMB = double.PositiveInfinity;
+
+            // add 100 documents
+            for (int i = 0; i < 100; i++)
+            {
+                AddDoc(writer);
+            }
+
+            // close
+            writer.Dispose();
+
+            long gen = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.IsTrue(gen > 0, "segment generation should be > 0 but got " + gen);
+
+            string[] files = dir.ListAll();
+            bool corrupted = false;
+            for (int i = 0; i < files.Length; i++)
+            {
+                if (files[i].EndsWith(".cfs"))
+                {
+                    dir.DeleteFile(files[i]);
+                    corrupted = true;
+                    break;
+                }
+            }
+            Assert.IsTrue(corrupted, "failed to find cfs file to remove");
+
+            IndexReader reader = null;
+            try
+            {
+                reader = DirectoryReader.Open(dir);
+                Assert.Fail("reader did not hit IOException on opening a corrupt index");
+            }
+#pragma warning disable 168
+            catch (Exception e)
+#pragma warning restore 168
+            {
+            }
+            if (reader != null)
+            {
+                reader.Dispose();
+            }
+            dir.Dispose();
+        }
+
+        // Simulate a writer that crashed while writing segments
+        // file: make sure we can still open the index (ie,
+        // gracefully fallback to the previous segments file),
+        // and that we can add to the index:
+        [Test]
+        public virtual void TestSimulatedCrashedWriter()
+        {
+            Directory dir = NewDirectory();
+            if (dir is MockDirectoryWrapper)
+            {
+                ((MockDirectoryWrapper)dir).PreventDoubleWrite = false;
+            }
+
+            IndexWriter writer = null;
+
+            writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+            // add 100 documents
+            for (int i = 0; i < 100; i++)
+            {
+                AddDoc(writer);
+            }
+
+            // close
+            writer.Dispose();
+
+            long gen = SegmentInfos.GetLastCommitGeneration(dir);
+            Assert.IsTrue(gen > 0, "segment generation should be > 0 but got " + gen);
+
+            // Make the next segments file, with last byte
+            // missing, to simulate a writer that crashed while
+            // writing segments file:
+            string fileNameIn = SegmentInfos.GetLastCommitSegmentsFileName(dir);
+            string fileNameOut = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen);
+            IndexInput @in = dir.OpenInput(fileNameIn, NewIOContext(Random()));
+            IndexOutput @out = dir.CreateOutput(fileNameOut, NewIOContext(Random()));
+            long length = @in.Length;
+            for (int i = 0; i < length - 1; i++)
+            {
+                @out.WriteByte(@in.ReadByte());
+            }
+            @in.Dispose();
+            @out.Dispose();
+
+            IndexReader reader = null;
+            try
+            {
+                reader = DirectoryReader.Open(dir);
+            }
+#pragma warning disable 168
+            catch (Exception e)
+#pragma warning restore 168
+            {
+                Assert.Fail("reader failed to open on a crashed index");
+            }
+            reader.Dispose();
+
+            try
+            {
+                writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE));
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e.StackTrace);
+                Assert.Fail("writer failed to open on a crashed index");
+            }
+
+            // add 100 documents
+            for (int i = 0; i < 100; i++)
+            {
+                AddDoc(writer);
+            }
+
+            // close
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestTermVectorExceptions()
+        {
+            FailOnTermVectors[] failures = new FailOnTermVectors[] { new FailOnTermVectors(FailOnTermVectors.AFTER_INIT_STAGE), new FailOnTermVectors(FailOnTermVectors.INIT_STAGE) };
+            int num = AtLeast(1);
+            for (int j = 0; j < num; j++)
+            {
+                foreach (FailOnTermVectors failure in failures)
+                {
+                    MockDirectoryWrapper dir = NewMockDirectory();
+                    IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+                    dir.FailOn(failure);
+                    int numDocs = 10 + Random().Next(30);
+                    for (int i = 0; i < numDocs; i++)
+                    {
+                        Document doc = new Document();
+                        Field field = NewTextField(Random(), "field", "a field", Field.Store.YES);
+                        doc.Add(field);
+                        // random TV
+                        try
+                        {
+                            w.AddDocument(doc);
+                            Assert.IsFalse(field.FieldType.StoreTermVectors);
+                        }
+                        catch (Exception e)
+                        {
+                            Assert.IsTrue(e.Message.StartsWith(FailOnTermVectors.EXC_MSG));
+                        }
+                        if (Random().Next(20) == 0)
+                        {
+                            w.Commit();
+                            TestUtil.CheckIndex(dir);
+                        }
+                    }
+                    Document document = new Document();
+                    document.Add(new TextField("field", "a field", Field.Store.YES));
+                    w.AddDocument(document);
+
+                    for (int i = 0; i < numDocs; i++)
+                    {
+                        Document doc = new Document();
+                        Field field = NewTextField(Random(), "field", "a field", Field.Store.YES);
+                        doc.Add(field);
+                        // random TV
+                        try
+                        {
+                            w.AddDocument(doc);
+                            Assert.IsFalse(field.FieldType.StoreTermVectors);
+                        }
+                        catch (Exception e)
+                        {
+                            Assert.IsTrue(e.Message.StartsWith(FailOnTermVectors.EXC_MSG));
+                        }
+                        if (Random().Next(20) == 0)
+                        {
+                            w.Commit();
+                            TestUtil.CheckIndex(dir);
+                        }
+                    }
+                    document = new Document();
+                    document.Add(new TextField("field", "a field", Field.Store.YES));
+                    w.AddDocument(document);
+                    w.Dispose();
+                    IndexReader reader = DirectoryReader.Open(dir);
+                    Assert.IsTrue(reader.NumDocs > 0);
+                    SegmentInfos sis = new SegmentInfos();
+                    sis.Read(dir);
+                    foreach (AtomicReaderContext context in reader.Leaves)
+                    {
+                        Assert.IsFalse((context.AtomicReader).FieldInfos.HasVectors);
+                    }
+                    reader.Dispose();
+                    dir.Dispose();
+                }
+            }
+        }
+
+        private class FailOnTermVectors : MockDirectoryWrapper.Failure
+        {
+            internal const string INIT_STAGE = "InitTermVectorsWriter";
+            internal const string AFTER_INIT_STAGE = "FinishDocument";
+            internal const string EXC_MSG = "FOTV";
+            internal readonly string Stage;
+
+            public FailOnTermVectors(string stage)
+            {
+                this.Stage = stage;
+            }
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                bool fail = StackTraceHelper.DoesStackTraceContainMethod(typeof(TermVectorsConsumer).Name, Stage);
+
+                if (fail)
+                {
+                    throw new Exception(EXC_MSG);
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestAddDocsNonAbortingException()
+        {
+            Directory dir = NewDirectory();
+            RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+            int numDocs1 = Random().Next(25);
+            for (int docCount = 0; docCount < numDocs1; docCount++)
+            {
+                Document doc = new Document();
+                doc.Add(NewTextField("content", "good content", Field.Store.NO));
+                w.AddDocument(doc);
+            }
+
+            IList<Document> docs = new List<Document>();
+            for (int docCount = 0; docCount < 7; docCount++)
+            {
+                Document doc = new Document();
+                docs.Add(doc);
+                doc.Add(NewStringField("id", docCount + "", Field.Store.NO));
+                doc.Add(NewTextField("content", "silly content " + docCount, Field.Store.NO));
+                if (docCount == 4)
+                {
+                    Field f = NewTextField("crash", "", Field.Store.NO);
+                    doc.Add(f);
+                    MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+                    tokenizer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+                    f.SetTokenStream(new CrashingFilter(this, "crash", tokenizer));
+                }
+            }
+            try
+            {
+                w.AddDocuments(docs);
+                // BUG: CrashingFilter didn't
+                Assert.Fail("did not hit expected exception");
+            }
+            catch (IOException ioe)
+            {
+                // expected
+                Assert.AreEqual(CRASH_FAIL_MESSAGE, ioe.Message);
+            }
+
+            int numDocs2 = Random().Next(25);
+            for (int docCount = 0; docCount < numDocs2; docCount++)
+            {
+                Document doc = new Document();
+                doc.Add(NewTextField("content", "good content", Field.Store.NO));
+                w.AddDocument(doc);
+            }
+
+            IndexReader r = w.Reader;
+            w.Dispose();
+
+            IndexSearcher s = NewSearcher(r);
+            PhraseQuery pq = new PhraseQuery();
+            pq.Add(new Term("content", "silly"));
+            pq.Add(new Term("content", "content"));
+            Assert.AreEqual(0, s.Search(pq, 1).TotalHits);
+
+            pq = new PhraseQuery();
+            pq.Add(new Term("content", "good"));
+            pq.Add(new Term("content", "content"));
+            Assert.AreEqual(numDocs1 + numDocs2, s.Search(pq, 1).TotalHits);
+            r.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestUpdateDocsNonAbortingException()
+        {
+            Directory dir = NewDirectory();
+            RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+            int numDocs1 = Random().Next(25);
+            for (int docCount = 0; docCount < numDocs1; docCount++)
+            {
+                Document doc = new Document();
+                doc.Add(NewTextField("content", "good content", Field.Store.NO));
+                w.AddDocument(doc);
+            }
+
+            // Use addDocs (no exception) to get docs in the index:
+            IList<Document> docs = new List<Document>();
+            int numDocs2 = Random().Next(25);
+            for (int docCount = 0; docCount < numDocs2; docCount++)
+            {
+                Document doc = new Document();
+                docs.Add(doc);
+                doc.Add(NewStringField("subid", "subs", Field.Store.NO));
+                doc.Add(NewStringField("id", docCount + "", Field.Store.NO));
+                doc.Add(NewTextField("content", "silly content " + docCount, Field.Store.NO));
+            }
+            w.AddDocuments(docs);
+
+            int numDocs3 = Random().Next(25);
+            for (int docCount = 0; docCount < numDocs3; docCount++)
+            {
+                Document doc = new Document();
+                doc.Add(NewTextField("content", "good content", Field.Store.NO));
+                w.AddDocument(doc);
+            }
+
+            docs.Clear();
+            int limit = TestUtil.NextInt(Random(), 2, 25);
+            int crashAt = Random().Next(limit);
+            for (int docCount = 0; docCount < limit; docCount++)
+            {
+                Document doc = new Document();
+                docs.Add(doc);
+                doc.Add(NewStringField("id", docCount + "", Field.Store.NO));
+                doc.Add(NewTextField("content", "silly content " + docCount, Field.Store.NO));
+                if (docCount == crashAt)
+                {
+                    Field f = NewTextField("crash", "", Field.Store.NO);
+                    doc.Add(f);
+                    MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+                    tokenizer.EnableChecks = false; // disable workflow checking as we forcefully close() in exceptional cases.
+                    f.SetTokenStream(new CrashingFilter(this, "crash", tokenizer));
+                }
+            }
+
+            try
+            {
+                w.UpdateDocuments(new Term("subid", "subs"), docs);
+                // BUG: CrashingFilter didn't
+                Assert.Fail("did not hit expected exception");
+            }
+            catch (IOException ioe)
+            {
+                // expected
+                Assert.AreEqual(CRASH_FAIL_MESSAGE, ioe.Message);
+            }
+
+            int numDocs4 = Random().Next(25);
+            for (int docCount = 0; docCount < numDocs4; docCount++)
+            {
+                Document doc = new Document();
+                doc.Add(NewTextField("content", "good content", Field.Store.NO));
+                w.AddDocument(doc);
+            }
+
+            IndexReader r = w.Reader;
+            w.Dispose();
+
+            IndexSearcher s = NewSearcher(r);
+            PhraseQuery pq = new PhraseQuery();
+            pq.Add(new Term("content", "silly"));
+            pq.Add(new Term("content", "content"));
+            Assert.AreEqual(numDocs2, s.Search(pq, 1).TotalHits);
+
+            pq = new PhraseQuery();
+            pq.Add(new Term("content", "good"));
+            pq.Add(new Term("content", "content"));
+            Assert.AreEqual(numDocs1 + numDocs3 + numDocs4, s.Search(pq, 1).TotalHits);
+            r.Dispose();
+            dir.Dispose();
+        }
+
+        internal class UOEDirectory : RAMDirectory
+        {
+            internal bool DoFail = false;
+
+            public override IndexInput OpenInput(string name, IOContext context)
+            {
+                if (DoFail
+                    && name.StartsWith("segments_")
+                    && StackTraceHelper.DoesStackTraceContainMethod("Read"))
+                {
+                    throw new NotSupportedException("expected UOE");
+                }
+
+                return base.OpenInput(name, context);
+            }
+        }
+
+        [Test]
+        public virtual void TestExceptionOnCtor()
+        {
+            UOEDirectory uoe = new UOEDirectory();
+            Directory d = new MockDirectoryWrapper(Random(), uoe);
+            IndexWriter iw = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
+            iw.AddDocument(new Document());
+            iw.Dispose();
+            uoe.DoFail = true;
+            try
+            {
+                new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
+                Assert.Fail("should have gotten a UOE");
+            }
+#pragma warning disable 168
+            catch (System.NotSupportedException expected)
+#pragma warning restore 168
+            {
+            }
+
+            uoe.DoFail = false;
+            d.Dispose();
+        }
+
+        [Test]
+        public virtual void TestIllegalPositions()
+        {
+            Directory dir = NewDirectory();
+            IndexWriter iw = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
+            Document doc = new Document();
+            Token t1 = new Token("foo", 0, 3);
+            t1.PositionIncrement = int.MaxValue;
+            Token t2 = new Token("bar", 4, 7);
+            t2.PositionIncrement = 200;
+            TokenStream overflowingTokenStream = new CannedTokenStream(new Token[] { t1, t2 });
+            Field field = new TextField("foo", overflowingTokenStream);
+            doc.Add(field);
+            try
+            {
+                iw.AddDocument(doc);
+                Assert.Fail();
+            }
+#pragma warning disable 168
+            catch (System.ArgumentException expected)
+#pragma warning restore 168
+            {
+                // expected exception
+            }
+            iw.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestLegalbutVeryLargePositions()
+        {
+            Directory dir = NewDirectory();
+            IndexWriter iw = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
+            Document doc = new Document();
+            Token t1 = new Token("foo", 0, 3);
+            t1.PositionIncrement = int.MaxValue - 500;
+            if (Random().NextBoolean())
+            {
+                t1.Payload = new BytesRef(new byte[] { 0x1 });
+            }
+            TokenStream overflowingTokenStream = new CannedTokenStream(new Token[] { t1 });
+            Field field = new TextField("foo", overflowingTokenStream);
+            doc.Add(field);
+            iw.AddDocument(doc);
+            iw.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestBoostOmitNorms()
+        {
+            Directory dir = NewDirectory();
+            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+            iwc.SetMergePolicy(NewLogMergePolicy());
+            IndexWriter iw = new IndexWriter(dir, iwc);
+            Document doc = new Document();
+            doc.Add(new StringField("field1", "sometext", Field.Store.YES));
+            doc.Add(new TextField("field2", "sometext", Field.Store.NO));
+            doc.Add(new StringField("foo", "bar", Field.Store.NO));
+            iw.AddDocument(doc); // add an 'ok' document
+            try
+            {
+                doc = new Document();
+                // try to boost with norms omitted
+                IList<IIndexableField> list = new List<IIndexableField>();
+                list.Add(new IndexableFieldAnonymousInnerClassHelper(this));
+                iw.AddDocument(list);
+                Assert.Fail("didn't get any exception, boost silently discarded");
+            }
+#pragma warning disable 168
+            catch (System.NotSupportedException expected)
+#pragma warning restore 168
+            {
+                // expected
+            }
+            DirectoryReader ir = DirectoryReader.Open(iw, false);
+            Assert.AreEqual(1, ir.NumDocs);
+            Assert.AreEqual("sometext", ir.Document(0).Get("field1"));
+            ir.Dispose();
+            iw.Dispose();
+            dir.Dispose();
+        }
+
+        private class IndexableFieldAnonymousInnerClassHelper : IIndexableField
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public IndexableFieldAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            public string Name
+            {
+                get { return "foo"; }
+            }
+
+            public IIndexableFieldType FieldType
+            {
+                get { return StringField.TYPE_NOT_STORED; }
+            }
+
+            public float Boost
+            {
+                get { return 5f; }
+            }
+
+            public BytesRef GetBinaryValue()
+            {
+                return null;
+            }
+
+            public string GetStringValue()
+            {
+                return "baz";
+            }
+
+            public TextReader GetReaderValue()
+            {
+                return null;
+            }
+
+            public object GetNumericValue()
+            {
+                return null;
+            }
+
+            public TokenStream GetTokenStream(Analyzer analyzer)
+            {
+                return null;
+            }
+        }
+
+        // See LUCENE-4870 TooManyOpenFiles errors are thrown as
+        // FNFExceptions which can trigger data loss.
+        [Test]
+        public virtual void TestTooManyFileException()
+        {
+            // Create failure that throws Too many open files exception randomly
+            MockDirectoryWrapper.Failure failure = new FailureAnonymousInnerClassHelper(this);
+
+            MockDirectoryWrapper dir = NewMockDirectory();
+            // The exception is only thrown on open input
+            dir.FailOnOpenInput = true;
+            dir.FailOn(failure);
+
+            // Create an index with one document
+            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+            IndexWriter iw = new IndexWriter(dir, iwc);
+            Document doc = new Document();
+            doc.Add(new StringField("foo", "bar", Field.Store.NO));
+            iw.AddDocument(doc); // add a document
+            iw.Commit();
+            DirectoryReader ir = DirectoryReader.Open(dir);
+            Assert.AreEqual(1, ir.NumDocs);
+            ir.Dispose();
+            iw.Dispose();
+
+            // Open and close the index a few times
+            for (int i = 0; i < 10; i++)
+            {
+                failure.SetDoFail();
+                iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+                try
+                {
+                    iw = new IndexWriter(dir, iwc);
+                }
+#pragma warning disable 168
+                catch (CorruptIndexException ex)
+#pragma warning restore 168
+                {
+                    // Exceptions are fine - we are running out of file handlers here
+                    continue;
+                }
+#pragma warning disable 168
+                catch (FileNotFoundException/* | NoSuchFileException*/ ex)
+#pragma warning restore 168
+                {
+                    continue;
+                }
+                failure.ClearDoFail();
+                iw.Dispose();
+                ir = DirectoryReader.Open(dir);
+                Assert.AreEqual(1, ir.NumDocs, "lost document after iteration: " + i);
+                ir.Dispose();
+            }
+
+            // Check if document is still there
+            failure.ClearDoFail();
+            ir = DirectoryReader.Open(dir);
+            Assert.AreEqual(1, ir.NumDocs);
+            ir.Dispose();
+
+            dir.Dispose();
+        }
+
+        private class FailureAnonymousInnerClassHelper : MockDirectoryWrapper.Failure
+        {
+            private readonly TestIndexWriterExceptions OuterInstance;
+
+            public FailureAnonymousInnerClassHelper(TestIndexWriterExceptions outerInstance)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            public override MockDirectoryWrapper.Failure Reset()
+            {
+                DoFail = false;
+                return this;
+            }
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                if (DoFail)
+                {
+                    if (Random().NextBoolean())
+                    {
+                        throw new FileNotFoundException("some/file/name.ext (Too many open files)");
+                    }
+                }
+            }
+        }
+
+        // Make sure if we hit a transient IOException (e.g., disk
+        // full), and then the exception stops (e.g., disk frees
+        // up), so we successfully close IW or open an NRT
+        // reader, we don't lose any deletes or updates:
+        [Test]
+        public virtual void TestNoLostDeletesOrUpdates()
+        {
+            int deleteCount = 0;
+            int docBase = 0;
+            int docCount = 0;
+
+            MockDirectoryWrapper dir = NewMockDirectory();
+            AtomicBoolean shouldFail = new AtomicBoolean();
+            dir.FailOn(new FailureAnonymousInnerClassHelper2(shouldFail));
+
+            RandomIndexWriter w = null;
+
+            for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
+            {
+                int numDocs = AtLeast(100);
+                if (VERBOSE)
+                {
+                    Console.WriteLine("\nTEST: iter=" + iter + " numDocs=" + numDocs + ".DocBase=" + docBase + " delCount=" + deleteCount);
+                }
+                if (w == null)
+                {
+                    IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+                    IMergeScheduler ms = iwc.MergeScheduler;
+                    if (ms is IConcurrentMergeScheduler)
+                    {
+                        IConcurrentMergeScheduler suppressFakeIOE = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this);
+
+                        IConcurrentMergeScheduler cms = (IConcurrentMergeScheduler)ms;
+                        suppressFakeIOE.SetMaxMergesAndThreads(cms.MaxMergeCount, cms.MaxThreadCount);
+                        suppressFakeIOE.SetMergeThreadPriority(cms.MergeThreadPriority);
+                        iwc.SetMergeScheduler(suppressFakeIOE);
+                    }
+
+                    w = new RandomIndexWriter(Random(), dir, iwc);
+                    // Since we hit exc during merging, a partial
+                    // forceMerge can easily return when there are still
+                    // too many segments in the index:
+                    w.DoRandomForceMergeAssert = false;
+                }
+                for (int i = 0; i < numDocs; i++)
+                {
+                    Document doc = new Document();
+                    doc.Add(new StringField("id", (docBase + i).ToString(), Field.Store.NO));
+                    if (DefaultCodecSupportsDocValues())
+                    {
+                        doc.Add(new NumericDocValuesField("f", 1L));
+                        doc.Add(new NumericDocValuesField("cf", 2L));
+                        doc.Add(new BinaryDocValuesField("bf", TestBinaryDocValuesUpdates.ToBytes(1L)));
+                        doc.Add(new BinaryDocValuesField("bcf", TestBinaryDocValuesUpdates.ToBytes(2L)));
+                    }
+                    w.AddDocument(doc);
+                }
+                docCount += numDocs;
+
+                // TODO: we could make the test more evil, by letting
+                // it throw more than one exc, randomly, before "recovering"
+
+                // TODO: we could also install an infoStream and try
+                // to fail in "more evil" places inside BDS
+
+                shouldFail.Set(true);
+                bool doClose = false;
+
+                try
+                {
+                    bool defaultCodecSupportsFieldUpdates = DefaultCodecSupportsFieldUpdates();
+                    for (int i = 0; i < numDocs; i++)
+                    {
+                        if (Random().Next(10) == 7)
+                        {
+                            bool fieldUpdate = defaultCodecSupportsFieldUpdates && Random().NextBoolean();
+                            if (fieldUpdate)
+                            {
+                                long value = iter;
+                                if (VERBOSE)
+                                {
+                                    Console.WriteLine("  update id=" + docBase + i + " to value " + value);
+                                }
+                                if (Random().NextBoolean()) // update only numeric field
+                                {
+                                    w.UpdateNumericDocValue(new Term("id", (docBase + i).ToString()), "f", value);
+                                    w.UpdateNumericDocValue(new Term("id", (docBase + i).ToString()), "cf", value * 2);
+                                }
+                                else if (Random().NextBoolean())
+                                {
+                                    w.UpdateBinaryDocValue(new Term("id", (docBase + i).ToString()), "bf", TestBinaryDocValuesUpdates.ToBytes(value));
+                                    w.UpdateBinaryDocValue(new Term("id", (docBase + i).ToString()), "bcf", TestBinaryDocValuesUpdates.ToBytes(value * 2));
+                                }
+                                else
+                                {
+                                    w.UpdateNumericDocValue(new Term("id", (docBase + i).ToString()), "f", value);
+                                    w.UpdateNumericDocValue(new Term("id", (docBase + i).ToString()), "cf", value * 2);
+                                    w.UpdateBinaryDocValue(new Term("id", (docBase + i).ToString()), "bf", TestBinaryDocValuesUpdates.ToBytes(value));
+                                    w.UpdateBinaryDocValue(new Term("id", (docBase + i).ToString()), "bcf", TestBinaryDocValuesUpdates.ToBytes(value * 2));
+                                }
+                            }
+
+                            // sometimes do both deletes and updates
+                            if (!fieldUpdate || Random().NextBoolean())
+                            {
+                                if (VERBOSE)
+                                {
+                                    Console.WriteLine("  delete id=" + (docBase + i).ToString());
+                                }
+                                deleteCount++;
+                                w.DeleteDocuments(new Term("id", "" + (docBase + i).ToString()));
+                            }
+                        }
+                    }
+
+                    // Trigger writeLiveDocs so we hit fake exc:
+                    IndexReader r = w.GetReader(true);
+
+                    // Sometimes we will make it here (we only randomly
+                    // throw the exc):
+                    Assert.AreEqual(docCount - deleteCount, r.NumDocs);
+        

<TRUNCATED>

Mime
View raw message