lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r756928 - in /incubator/lucene.net/trunk/C#/src/Test: Analysis/TestAnalyzers.cs Index/TestAddIndexesNoOptimize.cs Index/TestFieldsReader.cs Index/TestIndexModifier.cs Index/TestIndexWriter.cs Util/_TestUtil.cs
Date Sat, 21 Mar 2009 12:55:44 GMT
Author: digy
Date: Sat Mar 21 12:55:43 2009
New Revision: 756928

URL: http://svn.apache.org/viewvc?rev=756928&view=rev
Log:
LUCENENET-164 upgrade from version 2.3.1. to 2.3.2 (Lucene.Net Test + a small TestIndexModifier
patch)

Modified:
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestAddIndexesNoOptimize.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldsReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexModifier.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/_TestUtil.cs

Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestAnalyzers.cs?rev=756928&r1=756927&r2=756928&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs Sat Mar 21 12:55:43 2009
@@ -19,6 +19,7 @@
 
 using NUnit.Framework;
 
+using StandardTokenizer = Lucene.Net.Analysis.Standard.StandardTokenizer;
 using Payload = Lucene.Net.Index.Payload;
 using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 
@@ -109,6 +110,21 @@
 			ts = new BuffTokenFilter(ts);
 			VerifyPayload(ts);
 		}
+
+        // LUCENE-1150: Just a compile time test to ensure the
+        // StandardAnalyzer constants remain publicly accessible
+        public virtual void _TestStandardConstants()
+        {
+            int x = StandardTokenizer.ALPHANUM;
+            x = StandardTokenizer.APOSTROPHE;
+            x = StandardTokenizer.ACRONYM;
+            x = StandardTokenizer.COMPANY;
+            x = StandardTokenizer.EMAIL;
+            x = StandardTokenizer.HOST;
+            x = StandardTokenizer.NUM;
+            x = StandardTokenizer.CJ;
+            string[] y = StandardTokenizer.TOKEN_TYPES;
+        }
 	}
 	
 	class BuffTokenFilter : TokenFilter

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestAddIndexesNoOptimize.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestAddIndexesNoOptimize.cs?rev=756928&r1=756927&r2=756928&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestAddIndexesNoOptimize.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestAddIndexesNoOptimize.cs Sat Mar 21 12:55:43
2009
@@ -25,6 +25,7 @@
 using RAMDirectory = Lucene.Net.Store.RAMDirectory;
 using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
 
 namespace Lucene.Net.Index
 {
@@ -156,22 +157,6 @@
 			writer.Close();
 			
 			writer = NewWriter(dir, false);
-			int maxMergeDocs = writer.GetMaxMergeDocs();
-			writer.SetMaxMergeDocs(99);
-			
-			try
-			{
-				// upper bound cannot exceed maxMergeDocs
-				writer.AddIndexesNoOptimize(new Directory[]{aux});
-				Assert.IsTrue(false);
-			}
-			catch (System.ArgumentException)
-			{
-				Assert.AreEqual(100, writer.DocCount());
-				Assert.AreEqual(1, writer.GetSegmentCount());
-			}
-			
-			writer.SetMaxMergeDocs(maxMergeDocs);
 			try
 			{
 				// cannot add self
@@ -422,5 +407,45 @@
 			Assert.AreEqual(3, writer.GetSegmentCount());
 			writer.Close();
 		}
-	}
+
+        // LUCENE-1270
+        [Test]
+        public void TestHangOnClose()
+        {
+            Directory dir = new MockRAMDirectory();
+            IndexWriter writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), true);
+            writer.SetMergePolicy(new LogByteSizeMergePolicy());
+            writer.SetMaxBufferedDocs(5);
+            writer.SetUseCompoundFile(false);
+            writer.SetMergeFactor(100);
+
+            Document doc = new Document();
+            doc.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
+            for (int i = 0; i < 60; i++)
+                writer.AddDocument(doc);
+            writer.SetMaxBufferedDocs(200);
+            Document doc2 = new Document();
+            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
+            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
+            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
+            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES,
Field.Index.NO));
+            for (int i = 0; i < 10; i++)
+                writer.AddDocument(doc2);
+            writer.Close();
+
+            Directory dir2 = new MockRAMDirectory();
+            writer = new IndexWriter(dir2, false, new WhitespaceAnalyzer(), true);
+            LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
+            lmp.SetMinMergeMB(0.0001);
+            writer.SetMergePolicy(lmp);
+            writer.SetMergeFactor(4);
+            writer.SetUseCompoundFile(false);
+            writer.SetMergeScheduler(new SerialMergeScheduler());
+            writer.AddIndexesNoOptimize(new Directory[] { dir });
+            writer.Close();
+
+            dir.Close();
+            dir2.Close();
+        }
+    }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldsReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestFieldsReader.cs?rev=756928&r1=756927&r2=756928&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldsReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldsReader.cs Sat Mar 21 12:55:43 2009
@@ -27,6 +27,10 @@
 using Similarity = Lucene.Net.Search.Similarity;
 using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 using _TestUtil = Lucene.Net.Util._TestUtil;
+using IndexInput = Lucene.Net.Store.IndexInput;
+using IndexOutput = Lucene.Net.Store.IndexOutput;
+using Directory = Lucene.Net.Store.Directory;
+using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
 
 namespace Lucene.Net.Index
 {
@@ -331,5 +335,141 @@
 			Assert.AreEqual((byte) (size >> 8), sizebytes[2]);
 			Assert.AreEqual((byte) size, sizebytes[3]);
 		}
-	}
+
+        public class FaultyFSDirectory : Directory
+        {
+            FSDirectory fsDir;
+            public FaultyFSDirectory(System.IO.FileInfo dir)
+            {
+                fsDir = FSDirectory.GetDirectory(dir);
+                lockFactory = fsDir.GetLockFactory();
+            }
+            public override IndexInput OpenInput(String name)
+            {
+                return new FaultyIndexInput(fsDir.OpenInput(name));
+            }
+            public override string[] List()
+            {
+                return fsDir.List();
+            }
+            public override bool FileExists(string name)
+            {
+                return fsDir.FileExists(name);
+            }
+            public override long FileModified(string name)
+            {
+                return fsDir.FileModified(name);
+            }
+            public override void TouchFile(string name)
+            {
+                fsDir.TouchFile(name);
+            }
+            public override void DeleteFile(string name)
+            {
+                fsDir.DeleteFile(name);
+            }
+            public override void RenameFile(string name, string newName)
+            {
+                fsDir.RenameFile(name, newName);
+            }
+            public override long FileLength(string name)
+            {
+                return fsDir.FileLength(name);
+            }
+            public override IndexOutput CreateOutput(string name)
+            {
+                return fsDir.CreateOutput(name);
+            }
+            public override void Close()
+            {
+                fsDir.Close();
+            }
+        }
+
+        public class FaultyIndexInput : BufferedIndexInput
+        {
+            IndexInput indexInput;
+            internal static bool doFail;
+            int count;
+            public FaultyIndexInput(IndexInput indexInput)
+            {
+                this.indexInput = indexInput;
+            }
+            private void SimOutage()
+            {
+                if (doFail && count++ % 2 == 1)
+                {
+                    throw new System.IO.IOException("Simulated network outage");
+                }
+            }
+            protected override void ReadInternal(byte[] b, int offset, int length)
+            {
+                SimOutage();
+                indexInput.ReadBytes(b, offset, length);
+            }
+            protected override void SeekInternal(long pos)
+            {
+                //simOutage();
+                indexInput.Seek(pos);
+            }
+            public override long Length()
+            {
+                return indexInput.Length();
+            }
+            public override void Close()
+            {
+                indexInput.Close();
+            }
+        }
+
+        // LUCENE-1262
+        [Test]
+        public void TestExceptions()
+        {
+            string tempDir = SupportClass.AppSettings.Get("tempDir", "");
+            if (tempDir == null)
+                throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
+            System.IO.FileInfo indexDir = new System.IO.FileInfo(System.IO.Path.Combine(tempDir,
"testfieldswriterexceptions"));
+            try
+            {
+                Directory dir = new FaultyFSDirectory(indexDir);
+                IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+                for (int i = 0; i < 2; i++)
+                    writer.AddDocument(testDoc);
+                writer.Optimize();
+                writer.Close();
+                IndexReader reader = IndexReader.Open(dir);
+                FaultyIndexInput.doFail = true;
+                bool exc = false;
+                for (int i = 0; i < 2; i++)
+                {
+                    try
+                    {
+                        reader.Document(i);
+                    }
+                    catch (System.IO.IOException)
+                    {
+                        // expected
+                        exc = true;
+                    }
+                    try
+                    {
+                        reader.Document(i);
+                    }
+                    catch (System.IO.IOException)
+                    {
+                        // expected
+                        exc = true;
+                    }
+                }
+                Assert.IsTrue(exc);
+                reader.Close();
+                dir.Close();
+            }
+            finally
+            {
+                _TestUtil.RmDir(indexDir);
+            }
+        }
+    }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexModifier.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexModifier.cs?rev=756928&r1=756927&r2=756928&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexModifier.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexModifier.cs Sat Mar 21 12:55:43
2009
@@ -304,8 +304,11 @@
 					{
 						Lucene.Net.Documents.Document doc = GetDocument();
 						index.AddDocument(doc);
-						idStack.Add(doc.Get("id"));
-						added++;
+                        lock (idStack)
+                        {
+                            idStack.Add(doc.Get("id"));
+                            added++;
+                        }
 					}
 					else
 					{
@@ -314,8 +317,11 @@
 						System.String delId = null;
 						try
 						{
-							delId = idStack[idStack.Count - 1] as System.String;
-							idStack.RemoveAt(idStack.Count - 1);
+                            lock (idStack)
+                            {
+                                delId = idStack[idStack.Count - 1] as System.String;
+                                idStack.RemoveAt(idStack.Count - 1);
+                            }
 						}
 						catch (System.ArgumentOutOfRangeException)
 						{

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriter.cs?rev=756928&r1=756927&r2=756928&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs Sat Mar 21 12:55:43 2009
@@ -32,6 +32,7 @@
 using RAMDirectory = Lucene.Net.Store.RAMDirectory;
 using SingleInstanceLockFactory = Lucene.Net.Store.SingleInstanceLockFactory;
 using Analyzer = Lucene.Net.Analysis.Analyzer;
+using SinkTokenizer = Lucene.Net.Analysis.SinkTokenizer;
 using Token = Lucene.Net.Analysis.Token;
 using TokenFilter = Lucene.Net.Analysis.TokenFilter;
 using TokenStream = Lucene.Net.Analysis.TokenStream;
@@ -42,6 +43,9 @@
 using Hits = Lucene.Net.Search.Hits;
 using IndexSearcher = Lucene.Net.Search.IndexSearcher;
 using TermQuery = Lucene.Net.Search.TermQuery;
+using Query = Lucene.Net.Search.Query;
+using PhraseQuery = Lucene.Net.Search.PhraseQuery;
+using SpanTermQuery = Lucene.Net.Search.Spans.SpanTermQuery;
 using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 using _TestUtil = Lucene.Net.Util._TestUtil;
 
@@ -3525,5 +3529,225 @@
 			writer.AddDocument(doc);
 			writer.Close();
 		}
-	}
+
+        // LUCENE-1198
+        public class MockIndexWriter : IndexWriter
+        {
+            public MockIndexWriter(Directory dir, bool autoCommit, Analyzer a, bool create)
+                : base(dir, autoCommit, a, create)
+            {
+            }
+            internal bool doFail;
+            bool TestPoint(String name)
+            {
+                if (doFail && name.Equals("DocumentsWriter.ThreadState.init start"))
+                    throw new SystemException("intentionally failing");
+                return true;
+            }
+        }
+
+        [Test]
+        public void TestExceptionDocumentsWriterInit()
+        {
+            MockRAMDirectory dir = new MockRAMDirectory();
+            MockIndexWriter w = new MockIndexWriter(dir, false, new WhitespaceAnalyzer(),
true);
+            Document doc = new Document();
+            doc.Add(new Field("field", "a field", Field.Store.YES,
+                              Field.Index.TOKENIZED));
+            w.AddDocument(doc);
+            w.doFail = true;
+            try
+            {
+                w.AddDocument(doc);
+                Assert.Fail("did not hit exception");
+            }
+            catch (System.Exception)
+            {
+                // expected
+            }
+            w.Close();
+            _TestUtil.CheckIndex(dir);
+            dir.Close();
+        }
+
+
+        // LUCENE-1208
+        private class AnonymousClassAnalyzer3 : Analyzer
+        {
+            public AnonymousClassAnalyzer3(TestIndexWriter enclosingInstance)
+            {
+                InitBlock(enclosingInstance);
+            }
+            private void InitBlock(TestIndexWriter enclosingInstance)
+            {
+                this.enclosingInstance = enclosingInstance;
+            }
+            private TestIndexWriter enclosingInstance;
+            public TestIndexWriter Enclosing_Instance
+            {
+                get
+                {
+                    return enclosingInstance;
+                }
+
+            }
+            public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader
reader)
+            {
+                return new CrashingFilter(Enclosing_Instance, fieldName, new WhitespaceTokenizer(reader));
+            }
+        }
+
+        [Test]
+        public void TestExceptionJustBeforeFlush()
+        {
+            MockRAMDirectory dir = new MockRAMDirectory();
+            MockIndexWriter w = new MockIndexWriter(dir, false, new WhitespaceAnalyzer(),
true);
+            w.SetMaxBufferedDocs(2);
+            Document doc = new Document();
+            doc.Add(new Field("field", "a field", Field.Store.YES,
+                              Field.Index.TOKENIZED));
+            w.AddDocument(doc);
+            Analyzer analyzer = new AnonymousClassAnalyzer3(this);
+            Document crashDoc = new Document();
+            crashDoc.Add(new Field("crash", "do it on token 4", Field.Store.YES,
+                                   Field.Index.TOKENIZED));
+            try
+            {
+                w.AddDocument(crashDoc, analyzer);
+                Assert.Fail("did not hit expected exception");
+            }
+            catch (System.IO.IOException)
+            {
+                // expected
+            }
+            w.AddDocument(doc);
+            w.Close();
+            dir.Close();
+        }
+
+        // LUCENE-1210
+        public class MockIndexWriter2 : IndexWriter
+        {
+            public MockIndexWriter2(Directory dir, bool autoCommit, Analyzer a, bool create)
+                :
+              base(dir, autoCommit, a, create)
+            {
+            }
+            internal bool doFail;
+            internal bool failed;
+            protected override bool TestPoint(String name)
+            {
+                if (doFail && name.Equals("startMergeInit"))
+                {
+                    failed = true;
+                    throw new SystemException("intentionally failing");
+                }
+                return true;
+            }
+        }
+
+        [Test]
+        public void TestExceptionOnMergeInit()
+        {
+            MockRAMDirectory dir = new MockRAMDirectory();
+            MockIndexWriter2 w = new MockIndexWriter2(dir, false, new WhitespaceAnalyzer(),
true);
+            w.SetMaxBufferedDocs(2);
+            w.SetMergeFactor(2);
+            w.doFail = true;
+            w.SetMergeScheduler(new ConcurrentMergeScheduler());
+            Document doc = new Document();
+            doc.Add(new Field("field", "a field", Field.Store.YES,
+                              Field.Index.TOKENIZED));
+            for (int i = 0; i < 10; i++)
+                try
+                {
+                    w.AddDocument(doc);
+                }
+                catch (System.Exception)
+                {
+                    break;
+                }
+            ((ConcurrentMergeScheduler)w.GetMergeScheduler()).Sync();
+            Assert.IsTrue(w.failed);
+            w.Close();
+            dir.Close();
+        }
+
+        // LUCENE-1222
+        public class MockIndexWriter3 : IndexWriter
+        {
+            public MockIndexWriter3(Directory dir, bool autoCommit, Analyzer a, bool create)
+                :
+              base(dir, autoCommit, a, create)
+            {
+            }
+            internal bool wasCalled;
+            protected override void DoAfterFlush()
+            {
+                wasCalled = true;
+            }
+        }
+
+        [Test]
+        public void TestDoAfterFlush()
+        {
+            MockRAMDirectory dir = new MockRAMDirectory();
+            MockIndexWriter3 w = new MockIndexWriter3(dir, false, new WhitespaceAnalyzer(),
true);
+            Document doc = new Document();
+            doc.Add(new Field("field", "a field", Field.Store.YES,
+                              Field.Index.TOKENIZED));
+            w.AddDocument(doc);
+            w.Flush();
+            Assert.IsTrue(w.wasCalled);
+            w.wasCalled = true;
+            w.DeleteDocuments(new Term("field", "field"));
+            w.Flush();
+            Assert.IsTrue(w.wasCalled);
+            w.Close();
+            dir.Close();
+            IndexReader ir = IndexReader.Open(dir);
+            Assert.AreEqual(1, ir.MaxDoc());
+            Assert.AreEqual(0, ir.NumDocs());
+            ir.Close();
+        }
+
+        // LUCENE-1255
+        [Test]
+        public void TestNegativePositions()
+        {
+            SinkTokenizer tokens = new SinkTokenizer();
+            Token t = new Token();
+            t.SetTermText("a");
+            t.SetPositionIncrement(0);
+            tokens.Add(t);
+            t.SetTermText("b");
+            t.SetPositionIncrement(1);
+            tokens.Add(t);
+            t.SetTermText("c");
+            tokens.Add(t);
+            MockRAMDirectory dir = new MockRAMDirectory();
+            IndexWriter w = new IndexWriter(dir, false, new WhitespaceAnalyzer(), true);
+            Document doc = new Document();
+            doc.Add(new Field("field", tokens));
+            w.AddDocument(doc);
+            w.Close();
+            IndexSearcher s = new IndexSearcher(dir);
+            PhraseQuery pq = new PhraseQuery();
+            pq.Add(new Term("field", "a"));
+            pq.Add(new Term("field", "b"));
+            pq.Add(new Term("field", "c"));
+            Hits hits = s.Search(pq);
+            Assert.AreEqual(1, hits.Length());
+            Query q = new SpanTermQuery(new Term("field", "a"));
+            hits = s.Search(q);
+            Assert.AreEqual(1, hits.Length());
+            TermPositions tps = s.GetIndexReader().TermPositions(new Term("field", "a"));
+            Assert.IsTrue(tps.Next());
+            Assert.AreEqual(1, tps.Freq());
+            Assert.AreEqual(-1, tps.NextPosition());
+            Assert.IsTrue(_TestUtil.CheckIndex(dir));
+            s.Close();
+            dir.Close();
+        }
+    }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Util/_TestUtil.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Util/_TestUtil.cs?rev=756928&r1=756927&r2=756928&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Util/_TestUtil.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Util/_TestUtil.cs Sat Mar 21 12:55:43 2009
@@ -89,5 +89,18 @@
 			if (ms is ConcurrentMergeScheduler)
 				((ConcurrentMergeScheduler) ms).Sync();
 		}
+
+        public static bool CheckIndex(Lucene.Net.Store.Directory dir)
+        {
+            Lucene.Net.Index.CheckIndex.out_Renamed = new System.IO.StringWriter();
+            if (!Lucene.Net.Index.CheckIndex.Check(dir, false))
+            {
+                System.Console.WriteLine("CheckIndex failed");
+                System.Console.WriteLine(Lucene.Net.Index.CheckIndex.out_Renamed.ToString());
+                return false;
+            }
+            else
+                return true;
+        }
 	}
 }
\ No newline at end of file



Mime
View raw message