lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aro...@apache.org
Subject svn commit: r677059 [4/19] - in /incubator/lucene.net/trunk/C#/src: ./ Demo/DeleteFiles/ Demo/DemoLib/ Demo/IndexFiles/ Demo/IndexHtml/ Demo/SearchFiles/ Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Index/ Lucene.Net/Search/ Lucene.Net/Search/Function/ ...
Date Tue, 15 Jul 2008 21:44:10 GMT
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestBackwardsCompatibility.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestBackwardsCompatibility.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestBackwardsCompatibility.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestBackwardsCompatibility.cs Tue Jul 15 14:44:04 2008
@@ -19,24 +19,25 @@
 
 using NUnit.Framework;
 
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using Directory = Lucene.Net.Store.Directory;
+using FSDirectory = Lucene.Net.Store.FSDirectory;
 using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using Hits = Lucene.Net.Search.Hits;
 using IndexSearcher = Lucene.Net.Search.IndexSearcher;
 using TermQuery = Lucene.Net.Search.TermQuery;
-using Hits = Lucene.Net.Search.Hits;
-using Directory = Lucene.Net.Store.Directory;
-using FSDirectory = Lucene.Net.Store.FSDirectory;
-using Document = Lucene.Net.Documents.Document;
-using Field = Lucene.Net.Documents.Field;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 
 namespace Lucene.Net.Index
 {
 	
 	/*
-	Verify we can read the pre-XXX file format, do searches
+	Verify we can read the pre-2.1 file format, do searches
 	against it, and add documents to it.*/
 	
-    [TestFixture]
-	public class TestBackwardsCompatibility
+	[TestFixture]
+	public class TestBackwardsCompatibility : LuceneTestCase
 	{
 		
 		// Uncomment these cases & run in a pre-lockless checkout
@@ -54,20 +55,24 @@
 		
 		/* Unzips dirName + ".zip" --> dirName, removing dirName
 		first */
-		public virtual void  Unzip(System.String dirName)
+		public virtual void  Unzip(System.String zipName, System.String destDirName)
 		{
 #if SHARP_ZIP_LIB
-			RmDir(dirName);
-
+			// get zip input stream
 			ICSharpCode.SharpZipLib.Zip.ZipInputStream zipFile;
-            zipFile = new ICSharpCode.SharpZipLib.Zip.ZipInputStream(System.IO.File.OpenRead(dirName + ".zip"));
+			zipFile = new ICSharpCode.SharpZipLib.Zip.ZipInputStream(System.IO.File.OpenRead(zipName + ".zip"));
 
+			// get dest directory name
+			System.String dirName = FullDir(destDirName);
 			System.IO.FileInfo fileDir = new System.IO.FileInfo(dirName);
-			System.IO.Directory.CreateDirectory(fileDir.FullName);
 
-            ICSharpCode.SharpZipLib.Zip.ZipEntry entry;
+			// clean up old directory (if there) and create new directory
+			RmDir(fileDir.FullName);
+			System.IO.Directory.CreateDirectory(fileDir.FullName);
 
-            while ((entry = zipFile.GetNextEntry()) != null)
+			// copy file entries from zip stream to directory
+			ICSharpCode.SharpZipLib.Zip.ZipEntry entry;
+			while ((entry = zipFile.GetNextEntry()) != null)
 			{
 				System.IO.Stream streamout = new System.IO.BufferedStream(new System.IO.FileStream(new System.IO.FileInfo(System.IO.Path.Combine(fileDir.FullName, entry.Name)).FullName, System.IO.FileMode.Create));
 				
@@ -78,16 +83,16 @@
 					streamout.Write(buffer, 0, len);
 				}
 				
-                streamout.Close();
+				streamout.Close();
 			}
 			
 			zipFile.Close();
 #else
-            Assert.Fail("Needs integration with SharpZipLib");
+			Assert.Fail("Needs integration with SharpZipLib");
 #endif
 		}
 		
-        [Test]
+		[Test]
 		public virtual void  TestCreateCFS()
 		{
 			System.String dirName = "testindex.cfs";
@@ -95,7 +100,7 @@
 			RmDir(dirName);
 		}
 		
-        [Test]
+		[Test]
 		public virtual void  TestCreateNoCFS()
 		{
 			System.String dirName = "testindex.nocfs";
@@ -103,42 +108,49 @@
 			RmDir(dirName);
 		}
 		
-        [Test]
+		internal System.String[] oldNames = new System.String[]{"prelockless.cfs", "prelockless.nocfs", "presharedstores.cfs", "presharedstores.nocfs"};
+		
+		[Test]
 		public virtual void  TestSearchOldIndex()
 		{
-			System.String[] oldNames = new System.String[]{"prelockless.cfs", "prelockless.nocfs"};
 			for (int i = 0; i < oldNames.Length; i++)
 			{
-				System.String dirName = @"C#\src\test\index\index." + oldNames[i];
-				Unzip(dirName);
-				SearchIndex(dirName);
-				RmDir(dirName);
+				System.String dirName = @"Index\index." + oldNames[i];
+				Unzip(dirName, oldNames[i]);
+				SearchIndex(oldNames[i]);
+				RmDir(oldNames[i]);
 			}
 		}
 		
-        [Test]
+		[Test]
 		public virtual void  TestIndexOldIndexNoAdds()
 		{
-			System.String[] oldNames = new System.String[]{"prelockless.cfs", "prelockless.nocfs"};
 			for (int i = 0; i < oldNames.Length; i++)
 			{
-				System.String dirName = @"C#\src\test\index\index." + oldNames[i];
-				Unzip(dirName);
-				ChangeIndexNoAdds(dirName);
-				RmDir(dirName);
+				System.String dirName = @"Index\index." + oldNames[i];
+				Unzip(dirName, oldNames[i]);
+				ChangeIndexNoAdds(oldNames[i], true);
+				RmDir(oldNames[i]);
+				
+				Unzip(dirName, oldNames[i]);
+				ChangeIndexNoAdds(oldNames[i], false);
+				RmDir(oldNames[i]);
 			}
 		}
 		
-        [Test]
+		[Test]
 		public virtual void  TestIndexOldIndex()
 		{
-			System.String[] oldNames = new System.String[]{"prelockless.cfs", "prelockless.nocfs"};
 			for (int i = 0; i < oldNames.Length; i++)
 			{
-				System.String dirName = @"C#\src\test\index\index." + oldNames[i];
-				Unzip(dirName);
-				ChangeIndexWithAdds(dirName);
-				RmDir(dirName);
+				System.String dirName = @"Index\index." + oldNames[i];
+				Unzip(dirName, oldNames[i]);
+				ChangeIndexWithAdds(oldNames[i], true);
+				RmDir(oldNames[i]);
+				
+				Unzip(dirName, oldNames[i]);
+				ChangeIndexWithAdds(oldNames[i], false);
+				RmDir(oldNames[i]);
 			}
 		}
 		
@@ -147,12 +159,14 @@
 			//QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer());
 			//Query query = parser.parse("handle:1");
 			
+			dirName = FullDir(dirName);
+			
 			Directory dir = FSDirectory.GetDirectory(dirName);
 			IndexSearcher searcher = new IndexSearcher(dir);
 			
 			Hits hits = searcher.Search(new TermQuery(new Term("content", "aaa")));
 			Assert.AreEqual(34, hits.Length());
-			Lucene.Net.Documents.Document d = hits.Doc(0);
+			Document d = hits.Doc(0);
 			
 			// First document should be #21 since it's norm was increased:
 			Assert.AreEqual("21", d.Get("id"), "didn't get the right document first");
@@ -163,12 +177,15 @@
 		
 		/* Open pre-lockless index, add docs, do a delete &
 		* setNorm, and search */
-		public virtual void  ChangeIndexWithAdds(System.String dirName)
+		public virtual void  ChangeIndexWithAdds(System.String dirName, bool autoCommit)
 		{
 			
+			dirName = FullDir(dirName);
+			
 			Directory dir = FSDirectory.GetDirectory(dirName);
+			
 			// open writer
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+			IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
 			
 			// add 10 docs
 			for (int i = 0; i < 10; i++)
@@ -184,11 +201,11 @@
 			IndexSearcher searcher = new IndexSearcher(dir);
 			Hits hits = searcher.Search(new TermQuery(new Term("content", "aaa")));
 			Assert.AreEqual(44, hits.Length(), "wrong number of hits");
-			Lucene.Net.Documents.Document d = hits.Doc(0);
+			Document d = hits.Doc(0);
 			Assert.AreEqual("21", d.Get("id"), "wrong first document");
 			searcher.Close();
 			
-			// make sure we can do another delete & another setNorm against this
+			// make sure we can do delete & setNorm against this
 			// pre-lockless segment:
 			IndexReader reader = IndexReader.Open(dir);
 			Term searchTerm = new Term("id", "6");
@@ -197,7 +214,7 @@
 			reader.SetNorm(22, "content", (float) 2.0);
 			reader.Close();
 			
-			// make sure 2nd delete & 2nd norm "took":
+			// make sure they "took":
 			searcher = new IndexSearcher(dir);
 			hits = searcher.Search(new TermQuery(new Term("content", "aaa")));
 			Assert.AreEqual(43, hits.Length(), "wrong number of hits");
@@ -206,7 +223,7 @@
 			searcher.Close();
 			
 			// optimize
-			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+			writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
 			writer.Optimize();
 			writer.Close();
 			
@@ -222,20 +239,22 @@
 		
 		/* Open pre-lockless index, add docs, do a delete &
 		* setNorm, and search */
-		public virtual void  ChangeIndexNoAdds(System.String dirName)
+		public virtual void ChangeIndexNoAdds(System.String dirName, bool autoCommit)
 		{
 			
+			dirName = FullDir(dirName);
+			
 			Directory dir = FSDirectory.GetDirectory(dirName);
 			
 			// make sure searching sees right # hits
 			IndexSearcher searcher = new IndexSearcher(dir);
 			Hits hits = searcher.Search(new TermQuery(new Term("content", "aaa")));
 			Assert.AreEqual(34, hits.Length(), "wrong number of hits");
-			Lucene.Net.Documents.Document d = hits.Doc(0);
+			Document d = hits.Doc(0);
 			Assert.AreEqual("21", d.Get("id"), "wrong first document");
 			searcher.Close();
 			
-			// make sure we can do another delete & another setNorm against this
+			// make sure we can do a delete & setNorm against this
 			// pre-lockless segment:
 			IndexReader reader = IndexReader.Open(dir);
 			Term searchTerm = new Term("id", "6");
@@ -244,7 +263,7 @@
 			reader.SetNorm(22, "content", (float) 2.0);
 			reader.Close();
 			
-			// make sure 2nd delete & 2nd norm "took":
+			// make sure they "took":
 			searcher = new IndexSearcher(dir);
 			hits = searcher.Search(new TermQuery(new Term("content", "aaa")));
 			Assert.AreEqual(33, hits.Length(), "wrong number of hits");
@@ -253,7 +272,7 @@
 			searcher.Close();
 			
 			// optimize
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+			IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
 			writer.Optimize();
 			writer.Close();
 			
@@ -270,6 +289,10 @@
 		public virtual void  CreateIndex(System.String dirName, bool doCFS)
 		{
 			
+			RmDir(dirName);
+			
+			dirName = FullDir(dirName);
+			
 			Directory dir = FSDirectory.GetDirectory(dirName);
 			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
 			writer.SetUseCompoundFile(doCFS);
@@ -293,66 +316,85 @@
 		}
 		
 		/* Verifies that the expected file names were produced */
-		
-		// disable until hardcoded file names are fixes:
-        [Test]
+
+		[Test]
 		public virtual void  TestExactFileNames()
 		{
 			
-			System.String outputDir = "lucene.backwardscompat0.index";
-			Directory dir = FSDirectory.GetDirectory(outputDir);
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
-			for (int i = 0; i < 35; i++)
+			for (int pass = 0; pass < 2; pass++)
 			{
-				AddDoc(writer, i);
-			}
-			Assert.AreEqual(35, writer.DocCount(), "wrong doc count");
-			writer.Close();
-			
-			// Delete one doc so we get a .del file:
-			IndexReader reader = IndexReader.Open(dir);
-			Term searchTerm = new Term("id", "7");
-			int delCount = reader.DeleteDocuments(searchTerm);
-			Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
-			
-			// Set one norm so we get a .s0 file:
-			reader.SetNorm(21, "content", (float) 1.5);
-			reader.Close();
-			
-			// The numbering of fields can vary depending on which
-			// JRE is in use.  On some JREs we see content bound to
-			// field 0; on others, field 1.  So, here we have to
-			// figure out which field number corresponds to
-			// "content", and then set our expected file names below
-			// accordingly:
-			CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
-			FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
-			int contentFieldIndex = - 1;
-			for (int i = 0; i < fieldInfos.Size(); i++)
-			{
-				FieldInfo fi = fieldInfos.FieldInfo(i);
-				if (fi.Name.Equals("content"))
+				
+				System.String outputDir = "lucene.backwardscompat0.index";
+				RmDir(outputDir);
+				
+				try
 				{
-					contentFieldIndex = i;
-					break;
+					Directory dir = FSDirectory.GetDirectory(FullDir(outputDir));
+					
+					bool autoCommit = 0 == pass;
+					
+					IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+					writer.SetRAMBufferSizeMB(16.0);
+					//IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+					for (int i = 0; i < 35; i++)
+					{
+						AddDoc(writer, i);
+					}
+					Assert.AreEqual(35, writer.DocCount());
+					writer.Close();
+					
+					// Delete one doc so we get a .del file:
+					IndexReader reader = IndexReader.Open(dir);
+					Term searchTerm = new Term("id", "7");
+					int delCount = reader.DeleteDocuments(searchTerm);
+					Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
+					
+					// Set one norm so we get a .s0 file:
+					reader.SetNorm(21, "content", (float) 1.5);
+					reader.Close();
+					
+					// The numbering of fields can vary depending on which
+					// JRE is in use.  On some JREs we see content bound to
+					// field 0; on others, field 1.  So, here we have to
+					// figure out which field number corresponds to
+					// "content", and then set our expected file names below
+					// accordingly:
+					CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
+					FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
+					int contentFieldIndex = - 1;
+					for (int i = 0; i < fieldInfos.Size(); i++)
+					{
+						FieldInfo fi = fieldInfos.FieldInfo(i);
+						if (fi.Name_ForNUnitTest.Equals("content"))
+						{
+							contentFieldIndex = i;
+							break;
+						}
+					}
+					cfsReader.Close();
+					Assert.IsTrue(contentFieldIndex != - 1, "could not locate the 'content' field number in the _2.cfs segment");
+					
+					// Now verify file names:
+					System.String[] expected;
+					expected = new System.String[]{"_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_4", "segments.gen"};
+					
+					if (!autoCommit)
+						expected[3] = "segments_3";
+					
+					System.String[] actual = dir.List();
+					System.Array.Sort(expected);
+					System.Array.Sort(actual);
+					if (!ArrayEquals(expected, actual))
+					{
+						Assert.Fail("incorrect filenames in index: expected:\n    " + AsString(expected) + "\n  actual:\n    " + AsString(actual));
+					}
+					dir.Close();
+				}
+				finally
+				{
+					RmDir(outputDir);
 				}
 			}
-			cfsReader.Close();
-			Assert.IsTrue(contentFieldIndex != - 1, "could not locate the 'content' field number in the _2.cfs segment");
-			
-			// Now verify file names:
-			System.String[] expected = new System.String[]{"_0.cfs", "_0_1.del", "_1.cfs", "_2.cfs", "_2_1.s" + contentFieldIndex, "_3.cfs", "segments_a", "segments.gen"};
-			
-			System.String[] actual = dir.List();
-			System.Array.Sort(expected);
-			System.Array.Sort(actual);
-			if (!ArrayEquals(expected, actual))
-			{
-				Assert.Fail("incorrect filenames in index: expected:\n    " + AsString(expected) + "\n  actual:\n    " + AsString(actual));
-			}
-			dir.Close();
-			
-			RmDir(outputDir);
 		}
 		
 		private System.String AsString(System.String[] l)
@@ -379,7 +421,7 @@
 		
 		private void  RmDir(System.String dir)
 		{
-			System.IO.FileInfo fileDir = new System.IO.FileInfo(dir);
+			System.IO.FileInfo fileDir = new System.IO.FileInfo(FullDir(dir));
 			bool tmpBool;
 			if (System.IO.File.Exists(fileDir.FullName))
 				tmpBool = true;
@@ -387,7 +429,7 @@
 				tmpBool = System.IO.Directory.Exists(fileDir.FullName);
 			if (tmpBool)
 			{
-                System.String[] files = System.IO.Directory.GetFileSystemEntries(fileDir.FullName);
+				System.String[] files = System.IO.Directory.GetFileSystemEntries(fileDir.FullName);
 				if (files != null)
 				{
 					for (int i = 0; i < files.Length; i++)
@@ -424,29 +466,34 @@
 				bool generatedAux2 = tmpBool3;
 			}
 		}
+		
+		public static System.String FullDir(System.String dirName)
+		{
+			return new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), dirName)).FullName;
+		}
 
-        public static bool ArrayEquals(System.Array array1, System.Array array2)
-        {
-            bool result = false;
-            if ((array1 == null) && (array2 == null))
-                result = true;
-            else if ((array1 != null) && (array2 != null))
-            {
-                if (array1.Length == array2.Length)
-                {
-                    int length = array1.Length;
-                    result = true;
-                    for (int index = 0; index < length; index++)
-                    {
-                        if (!(array1.GetValue(index).Equals(array2.GetValue(index))))
-                        {
-                            result = false;
-                            break;
-                        }
-                    }
-                }
-            }
-            return result;
-        }
-    }
+		public static bool ArrayEquals(System.Array array1, System.Array array2)
+		{
+			bool result = false;
+			if ((array1 == null) && (array2 == null))
+				result = true;
+			else if ((array1 != null) && (array2 != null))
+			{
+				if (array1.Length == array2.Length)
+				{
+					int length = array1.Length;
+					result = true;
+					for (int index = 0; index < length; index++)
+					{
+						if (!(array1.GetValue(index).Equals(array2.GetValue(index))))
+						{
+							result = false;
+							break;
+						}
+					}
+				}
+			}
+			return result;
+		}
+	}
 }
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestCheckIndex.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestCheckIndex.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestCheckIndex.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestCheckIndex.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+
+namespace Lucene.Net.Index
+{
+	
+	[TestFixture]
+	public class TestCheckIndex : LuceneTestCase
+	{
+		
+		[Test]
+		public virtual void  TestDeletedDocs()
+		{
+			MockRAMDirectory dir = new MockRAMDirectory();
+			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+			writer.SetMaxBufferedDocs(2);
+			Document doc = new Document();
+			doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+			for (int i = 0; i < 19; i++)
+			{
+				writer.AddDocument(doc);
+			}
+			writer.Close();
+			IndexReader reader = IndexReader.Open(dir);
+			reader.DeleteDocument(5);
+			reader.Close();
+			
+			CheckIndex.out_Renamed = new System.IO.StringWriter();
+			bool condition = CheckIndex.Check(dir, false);
+			String message = CheckIndex.out_Renamed.ToString();
+			Assert.IsTrue(condition, message);
+		}
+	}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestCompoundFile.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestCompoundFile.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestCompoundFile.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestCompoundFile.cs Tue Jul 15 14:44:04 2008
@@ -19,22 +19,24 @@
 
 using NUnit.Framework;
 
+//using TestRunner = junit.textui.TestRunner;
 using Directory = Lucene.Net.Store.Directory;
 using FSDirectory = Lucene.Net.Store.FSDirectory;
 using IndexInput = Lucene.Net.Store.IndexInput;
 using IndexOutput = Lucene.Net.Store.IndexOutput;
 using _TestHelper = Lucene.Net.Store._TestHelper;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using _TestUtil = Lucene.Net.Util._TestUtil;
 
 namespace Lucene.Net.Index
 {
 	
 	
-	/// <author>  dmitrys@earthlink.net
-	/// </author>
-	/// <version>  $Id: TestCompoundFile.java 208807 2005-07-01 22:13:53Z dnaber $
+	/// <summary> </summary>
+	/// <version>  $Id: TestCompoundFile.java 602165 2007-12-07 17:42:33Z mikemccand $
 	/// </version>
 	[TestFixture]
-    public class TestCompoundFile
+	public class TestCompoundFile : LuceneTestCase
 	{
 		/// <summary>Main for running test case by itself. </summary>
 		[STAThread]
@@ -58,12 +60,12 @@
 		private Directory dir;
 		
 		[SetUp]
-		public virtual void  SetUp()
+		public override void SetUp()
 		{
-            System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "testIndex"));
-            Lucene.Net.Util._TestUtil.RmDir(file);
-            dir = FSDirectory.GetDirectory(file);
-        }
+			System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "testIndex"));
+			Lucene.Net.Util._TestUtil.RmDir(file);
+			dir = FSDirectory.GetDirectory(file);
+		}
 		
 		
 		/// <summary>Creates a file of the specified size with random data. </summary>
@@ -177,7 +179,7 @@
 		/// Files of different sizes are tested: 0, 1, 10, 100 bytes.
 		/// </summary>
 		[Test]
-        public virtual void  TestSingleFile()
+		public virtual void  TestSingleFile()
 		{
 			int[] data = new int[]{0, 1, 10, 100};
 			for (int i = 0; i < data.Length; i++)
@@ -204,7 +206,7 @@
 		/// 
 		/// </summary>
 		[Test]
-        public virtual void  TestTwoFiles()
+		public virtual void  TestTwoFiles()
 		{
 			CreateSequenceFile(dir, "d1", (byte) 0, 15);
 			CreateSequenceFile(dir, "d2", (byte) 0, 114);
@@ -238,7 +240,7 @@
 		/// the length of the buffer used internally by the compound file logic.
 		/// </summary>
 		[Test]
-        public virtual void  TestRandomFiles()
+		public virtual void  TestRandomFiles()
 		{
 			// Setup the test segment
 			System.String segment = "test";
@@ -337,7 +339,7 @@
 				b = in_Renamed.ReadByte();
 				Assert.Fail("expected readByte() to throw exception");
 			}
-			catch (System.Exception e)
+			catch (System.Exception)
 			{
 				// expected exception
 			}
@@ -426,7 +428,7 @@
 		/// their file positions are independent of each other.
 		/// </summary>
 		[Test]
-        public virtual void  TestRandomAccess()
+		public virtual void  TestRandomAccess()
 		{
 			SetUp_2();
 			CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
@@ -507,7 +509,7 @@
 		/// their file positions are independent of each other.
 		/// </summary>
 		[Test]
-        public virtual void  TestRandomAccessClones()
+		public virtual void  TestRandomAccessClones()
 		{
 			SetUp_2();
 			CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
@@ -596,7 +598,7 @@
 				IndexInput e1 = cr.OpenInput("bogus");
 				Assert.Fail("File not found");
 			}
-			catch (System.IO.IOException e)
+			catch (System.IO.IOException)
 			{
 				/* success */
 				//System.out.println("SUCCESS: File Not Found: " + e);
@@ -620,7 +622,7 @@
 				byte test = is_Renamed.ReadByte();
 				Assert.Fail("Single byte read past end of file");
 			}
-			catch (System.IO.IOException e)
+			catch (System.IO.IOException)
 			{
 				/* success */
 				//System.out.println("SUCCESS: single byte read past end of file: " + e);
@@ -632,7 +634,7 @@
 				is_Renamed.ReadBytes(b, 0, 50);
 				Assert.Fail("Block read past end of file");
 			}
-			catch (System.IO.IOException e)
+			catch (System.IO.IOException)
 			{
 				/* success */
 				//System.out.println("SUCCESS: block read past end of file: " + e);
@@ -642,31 +644,31 @@
 			cr.Close();
 		}
 
-        /// <summary>This test that writes larger than the size of the buffer output
-        /// will correctly increment the file pointer.
-        /// </summary>
-        [Test]
-        public virtual void  TestLargeWrites()
-                                               {
-            IndexOutput os = dir.CreateOutput("testBufferStart.txt");
+		/// <summary>This test that writes larger than the size of the buffer output
+		/// will correctly increment the file pointer.
+		/// </summary>
+		[Test]
+		public virtual void  TestLargeWrites()
+											   {
+			IndexOutput os = dir.CreateOutput("testBufferStart.txt");
 
-            byte[] largeBuf = new byte[2048];
-            for (int i=0; i<largeBuf.Length; i++)
-            {
-                largeBuf[i] = (byte) ((new System.Random().NextDouble()) * 256);
-            }
+			byte[] largeBuf = new byte[2048];
+			for (int i=0; i<largeBuf.Length; i++)
+			{
+				largeBuf[i] = (byte) ((new System.Random().NextDouble()) * 256);
+			}
 
-            long currentPos = os.GetFilePointer();
-            os.WriteBytes(largeBuf, largeBuf.Length);
+			long currentPos = os.GetFilePointer();
+			os.WriteBytes(largeBuf, largeBuf.Length);
 
-            try
-            {
-                Assert.AreEqual(currentPos + largeBuf.Length, os.GetFilePointer());
-            }
-            finally
-            {
-                os.Close();
-            }
-        }
- 	}
+			try
+			{
+				Assert.AreEqual(currentPos + largeBuf.Length, os.GetFilePointer());
+			}
+			finally
+			{
+				os.Close();
+			}
+		}
+	}
 }
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestConcurrentMergeScheduler.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestConcurrentMergeScheduler.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestConcurrentMergeScheduler.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestConcurrentMergeScheduler.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using SimpleAnalyzer = Lucene.Net.Analysis.SimpleAnalyzer;
+
+namespace Lucene.Net.Index
+{
+	
+	[TestFixture]
+	public class TestConcurrentMergeScheduler : LuceneTestCase
+	{
+		
+		private static readonly Analyzer ANALYZER = new SimpleAnalyzer();
+		
+		private class FailOnlyOnFlush : MockRAMDirectory.Failure
+		{
+			new internal bool doFail = false;
+			
+			public override void  SetDoFail()
+			{
+				this.doFail = true;
+			}
+			public override void  ClearDoFail()
+			{
+				this.doFail = false;
+			}
+
+			public override void Eval(MockRAMDirectory dir)
+			{
+				if (doFail)
+				{
+					// {{DOUG-2.3.1}} this code is suspect.  i have preserved the original (below) for 
+					// comparative purposes.
+					if (new System.Exception().StackTrace.Contains("doFlush"))
+						throw new System.IO.IOException("now failing during flush");
+					//StackTraceElement[] trace = new System.Exception().getStackTrace();
+					//for (int i = 0; i < trace.Length; i++)
+					//{
+					//    if ("doFlush".Equals(trace[i].getMethodName()))
+					//    {
+					//        //new RuntimeException().printStackTrace(System.out);
+					//        throw new System.IO.IOException("now failing during flush");
+					//    }
+					//}
+				}
+			}
+		}
+		
+		// Make sure running BG merges still work fine even when
+		// we are hitting exceptions during flushing.
+		[Test]
+		public virtual void  TestFlushExceptions()
+		{
+			
+			MockRAMDirectory directory = new MockRAMDirectory();
+			FailOnlyOnFlush failure = new FailOnlyOnFlush();
+			directory.FailOn(failure);
+			
+			IndexWriter writer = new IndexWriter(directory, ANALYZER, true);
+			ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+			writer.SetMergeScheduler(cms);
+			writer.SetMaxBufferedDocs(2);
+			Document doc = new Document();
+			Field idField = new Field("id", "", Field.Store.YES, Field.Index.UN_TOKENIZED);
+			doc.Add(idField);
+			for (int i = 0; i < 10; i++)
+			{
+				for (int j = 0; j < 20; j++)
+				{
+					idField.SetValue(System.Convert.ToString(i * 20 + j));
+					writer.AddDocument(doc);
+				}
+				
+				writer.AddDocument(doc);
+				
+				failure.SetDoFail();
+				try
+				{
+					writer.Flush();
+					Assert.Fail("failed to hit IOException");
+				}
+				catch (System.IO.IOException)
+				{
+					failure.ClearDoFail();
+				}
+			}
+			
+			writer.Close();
+			IndexReader reader = IndexReader.Open(directory);
+			Assert.AreEqual(200, reader.NumDocs());
+			reader.Close();
+			directory.Close();
+		}
+		
+		// Test that deletes committed after a merge started and
+		// before it finishes, are correctly merged back:
+		[Test]
+		public virtual void  TestDeleteMerging()
+		{
+			
+			RAMDirectory directory = new MockRAMDirectory();
+			
+			IndexWriter writer = new IndexWriter(directory, ANALYZER, true);
+			ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+			writer.SetMergeScheduler(cms);
+			
+			LogDocMergePolicy mp = new LogDocMergePolicy();
+			writer.SetMergePolicy(mp);
+			
+			// Force degenerate merging so we can get a mix of
+			// merging of segments with and without deletes at the
+			// start:
+			mp.SetMinMergeDocs(1000);
+			
+			Document doc = new Document();
+			Field idField = new Field("id", "", Field.Store.YES, Field.Index.UN_TOKENIZED);
+			doc.Add(idField);
+			for (int i = 0; i < 10; i++)
+			{
+				for (int j = 0; j < 100; j++)
+				{
+					idField.SetValue(System.Convert.ToString(i * 100 + j));
+					writer.AddDocument(doc);
+				}
+				
+				int delID = i;
+				while (delID < 100 * (1 + i))
+				{
+					writer.DeleteDocuments(new Term("id", "" + delID));
+					delID += 10;
+				}
+				
+				writer.Flush();
+			}
+			
+			writer.Close();
+			IndexReader reader = IndexReader.Open(directory);
+			// Verify that we did not lose any deletes...
+			Assert.AreEqual(450, reader.NumDocs());
+			reader.Close();
+			directory.Close();
+		}
+		
+		[Test]
+		public virtual void  TestNoExtraFiles()
+		{
+			
+			RAMDirectory directory = new MockRAMDirectory();
+			
+			for (int pass = 0; pass < 2; pass++)
+			{
+				
+				bool autoCommit = pass == 0;
+				IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
+				
+				for (int iter = 0; iter < 7; iter++)
+				{
+					ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+					writer.SetMergeScheduler(cms);
+					writer.SetMaxBufferedDocs(2);
+					
+					for (int j = 0; j < 21; j++)
+					{
+						Document doc = new Document();
+						doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.TOKENIZED));
+						writer.AddDocument(doc);
+					}
+					
+					writer.Close();
+					TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles autoCommit=" + autoCommit);
+					
+					// Reopen
+					writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
+				}
+				
+				writer.Close();
+			}
+			
+			directory.Close();
+		}
+		
+		[Test]
+		public virtual void  TestNoWaitClose()
+		{
+			RAMDirectory directory = new MockRAMDirectory();
+			
+			Document doc = new Document();
+			Field idField = new Field("id", "", Field.Store.YES, Field.Index.UN_TOKENIZED);
+			doc.Add(idField);
+			
+			for (int pass = 0; pass < 2; pass++)
+			{
+				bool autoCommit = pass == 0;
+				IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
+				
+				for (int iter = 0; iter < 10; iter++)
+				{
+					ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+					writer.SetMergeScheduler(cms);
+					writer.SetMaxBufferedDocs(2);
+					writer.SetMergeFactor(100);
+					
+					for (int j = 0; j < 201; j++)
+					{
+						idField.SetValue(System.Convert.ToString(iter * 201 + j));
+						writer.AddDocument(doc);
+					}
+					
+					int delID = iter * 201;
+					for (int j = 0; j < 20; j++)
+					{
+						writer.DeleteDocuments(new Term("id", System.Convert.ToString(delID)));
+						delID += 5;
+					}
+					
+					// Force a bunch of merge threads to kick off so we
+					// stress out aborting them on close:
+					writer.SetMergeFactor(3);
+					writer.AddDocument(doc);
+					writer.Flush();
+					
+					writer.Close(false);
+					
+					IndexReader reader = IndexReader.Open(directory);
+					Assert.AreEqual((1 + iter) * 182, reader.NumDocs());
+					reader.Close();
+					
+					// Reopen
+					writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
+				}
+				writer.Close();
+			}
+			
+			directory.Close();
+		}
+	}
+}
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestDeletionPolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestDeletionPolicy.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestDeletionPolicy.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestDeletionPolicy.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,790 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using Directory = Lucene.Net.Store.Directory;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using Hits = Lucene.Net.Search.Hits;
+using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+using Query = Lucene.Net.Search.Query;
+using TermQuery = Lucene.Net.Search.TermQuery;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+namespace Lucene.Net.Index
+{
+	
+	/*
+	Verify we can read the pre-2.1 file format, do searches
+	against it, and add documents to it.*/
+	
+	[TestFixture]
+	public class TestDeletionPolicy : LuceneTestCase
+	{
+		private void  VerifyCommitOrder(System.Collections.IList commits)
+		{
+			long last = SegmentInfos.GenerationFromSegmentsFileName(((IndexCommitPoint) commits[0]).GetSegmentsFileName());
+			for (int i = 1; i < commits.Count; i++)
+			{
+				long now = SegmentInfos.GenerationFromSegmentsFileName(((IndexCommitPoint) commits[i]).GetSegmentsFileName());
+				Assert.IsTrue(now > last, "SegmentInfos commits are out-of-order");
+				last = now;
+			}
+		}
+		
+		internal class KeepAllDeletionPolicy : IndexDeletionPolicy
+		{
+			public KeepAllDeletionPolicy(TestDeletionPolicy enclosingInstance)
+			{
+				InitBlock(enclosingInstance);
+			}
+			private void  InitBlock(TestDeletionPolicy enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private TestDeletionPolicy enclosingInstance;
+			public TestDeletionPolicy Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			internal int numOnInit;
+			internal int numOnCommit;
+			public virtual void  OnInit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				numOnInit++;
+			}
+			public virtual void  OnCommit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				numOnCommit++;
+			}
+		}
+		
+		/// <summary> This is useful for adding to a big index w/ autoCommit
+		/// false when you know readers are not using it.
+		/// </summary>
+		internal class KeepNoneOnInitDeletionPolicy : IndexDeletionPolicy
+		{
+			public KeepNoneOnInitDeletionPolicy(TestDeletionPolicy enclosingInstance)
+			{
+				InitBlock(enclosingInstance);
+			}
+			private void  InitBlock(TestDeletionPolicy enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private TestDeletionPolicy enclosingInstance;
+			public TestDeletionPolicy Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			internal int numOnInit;
+			internal int numOnCommit;
+			public virtual void  OnInit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				numOnInit++;
+				// On init, delete all commit points:
+				System.Collections.IEnumerator it = commits.GetEnumerator();
+				while (it.MoveNext())
+				{
+					((IndexCommitPoint) it.Current).Delete();
+				}
+			}
+			public virtual void  OnCommit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				int size = commits.Count;
+				// Delete all but last one:
+				for (int i = 0; i < size - 1; i++)
+				{
+					((IndexCommitPoint) commits[i]).Delete();
+				}
+				numOnCommit++;
+			}
+		}
+		
+		internal class KeepLastNDeletionPolicy : IndexDeletionPolicy
+		{
+			private void  InitBlock(TestDeletionPolicy enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private TestDeletionPolicy enclosingInstance;
+			public TestDeletionPolicy Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			internal int numOnInit;
+			internal int numOnCommit;
+			internal int numToKeep;
+			internal int numDelete;
+			internal System.Collections.Hashtable seen = new System.Collections.Hashtable();
+			
+			public KeepLastNDeletionPolicy(TestDeletionPolicy enclosingInstance, int numToKeep)
+			{
+				InitBlock(enclosingInstance);
+				this.numToKeep = numToKeep;
+			}
+			
+			public virtual void  OnInit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				numOnInit++;
+				// do no deletions on init
+				DoDeletes(commits, false);
+			}
+			
+			public virtual void  OnCommit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				DoDeletes(commits, true);
+			}
+			
+			private void  DoDeletes(System.Collections.IList commits, bool isCommit)
+			{
+				
+				// Assert that we really are only called for each new
+				// commit:
+				if (isCommit)
+				{
+					System.String fileName = ((IndexCommitPoint) commits[commits.Count - 1]).GetSegmentsFileName();
+					if (seen.Contains(fileName))
+					{
+						throw new System.SystemException("onCommit was called twice on the same commit point: " + fileName);
+					}
+					seen.Add(fileName, fileName);
+					numOnCommit++;
+				}
+				int size = commits.Count;
+				for (int i = 0; i < size - numToKeep; i++)
+				{
+					((IndexCommitPoint) commits[i]).Delete();
+					numDelete++;
+				}
+			}
+		}
+		
+		/*
+		* Delete a commit only when it has been obsoleted by N
+		* seconds.
+		*/
+		internal class ExpirationTimeDeletionPolicy : IndexDeletionPolicy
+		{
+			private void  InitBlock(TestDeletionPolicy enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private TestDeletionPolicy enclosingInstance;
+			public TestDeletionPolicy Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			
+			internal Directory dir;
+			internal double expirationTimeSeconds;
+			internal int numDelete;
+			
+			public ExpirationTimeDeletionPolicy(TestDeletionPolicy enclosingInstance, Directory dir, double seconds)
+			{
+				InitBlock(enclosingInstance);
+				this.dir = dir;
+				this.expirationTimeSeconds = seconds;
+			}
+			
+			public virtual void  OnInit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				OnCommit(commits);
+			}
+			
+			public virtual void  OnCommit(System.Collections.IList commits)
+			{
+				Enclosing_Instance.VerifyCommitOrder(commits);
+				
+				IndexCommitPoint lastCommit = (IndexCommitPoint) commits[commits.Count - 1];
+				
+				// Any commit older than expireTime should be deleted:
+				double expireTime = dir.FileModified(lastCommit.GetSegmentsFileName()) / 1000.0 - expirationTimeSeconds;
+				
+				System.Collections.IEnumerator it = commits.GetEnumerator();
+				
+				while (it.MoveNext())
+				{
+					IndexCommitPoint commit = (IndexCommitPoint) it.Current;
+					double modTime = dir.FileModified(commit.GetSegmentsFileName()) / 1000.0;
+					if (commit != lastCommit && modTime < expireTime)
+					{
+						commit.Delete();
+						numDelete += 1;
+					}
+				}
+			}
+		}
+		
+		/*
+		* Test "by time expiration" deletion policy:
+		*/
+		[Test]
+		public virtual void  TestExpirationTimeDeletionPolicy()
+		{
+			
+			double SECONDS = 2.0;
+			
+			bool autoCommit = false;
+			bool useCompoundFile = true;
+			
+			Directory dir = new RAMDirectory();
+			ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(this, dir, SECONDS);
+			IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+			writer.SetUseCompoundFile(useCompoundFile);
+			writer.Close();
+			
+			long lastDeleteTime = 0;
+			for (int i = 0; i < 7; i++)
+			{
+				// Record last time when writer performed deletes of
+				// past commits
+				lastDeleteTime = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
+				writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+				writer.SetUseCompoundFile(useCompoundFile);
+				for (int j = 0; j < 17; j++)
+				{
+					AddDoc(writer);
+				}
+				writer.Close();
+				
+				// Make sure to sleep long enough so that some commit
+				// points will be deleted:
+				System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * (int) (1000.0 * (SECONDS / 5.0))));
+			}
+			
+			// First, make sure the policy in fact deleted something:
+			Assert.IsTrue(policy.numDelete > 0, "no commits were deleted");
+			
+			// Then simplistic check: just verify that the
+			// segments_N's that still exist are in fact within SECONDS
+			// seconds of the last one's mod time, and, that I can
+			// open a reader on each:
+			long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
+			
+			System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
+			dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
+			while (gen > 0)
+			{
+				try
+				{
+					IndexReader reader = IndexReader.Open(dir);
+					reader.Close();
+					fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
+					long modTime = dir.FileModified(fileName);
+					Assert.IsTrue(lastDeleteTime - modTime <= (SECONDS * 1000), "commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted");
+				}
+				catch (System.IO.IOException)
+				{
+					// OK
+					break;
+				}
+				
+				dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+				gen--;
+			}
+			
+			dir.Close();
+		}
+		
+		/*
+		* Test a silly deletion policy that keeps all commits around.
+		*/
+		[Test]
+		public virtual void  TestKeepAllDeletionPolicy()
+		{
+			
+			for (int pass = 0; pass < 4; pass++)
+			{
+				
+				bool autoCommit = pass < 2;
+				bool useCompoundFile = (pass % 2) > 0;
+				
+				KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
+				
+				Directory dir = new RAMDirectory();
+				
+				IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+				writer.SetMaxBufferedDocs(10);
+				writer.SetUseCompoundFile(useCompoundFile);
+				for (int i = 0; i < 107; i++)
+				{
+					AddDoc(writer);
+				}
+				writer.Close();
+				
+				writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+				writer.SetUseCompoundFile(useCompoundFile);
+				writer.Optimize();
+				writer.Close();
+				
+				Assert.AreEqual(2, policy.numOnInit);
+				if (autoCommit)
+				{
+					Assert.IsTrue(policy.numOnCommit > 2);
+				}
+				else
+				{
+					// If we are not auto committing then there should
+					// be exactly 2 commits (one per close above):
+					Assert.AreEqual(2, policy.numOnCommit);
+				}
+				
+				// Simplistic check: just verify all segments_N's still
+				// exist, and, I can open a reader on each:
+				dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
+				long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
+				while (gen > 0)
+				{
+					IndexReader reader = IndexReader.Open(dir);
+					reader.Close();
+					dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+					gen--;
+					
+					if (gen > 0)
+					{
+						// Now that we've removed a commit point, which
+						// should have orphan'd at least one index file.
+						// Open & close a writer and assert that it
+						// actually removed something:
+						int preCount = dir.List().Length;
+						writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false, policy);
+						writer.Close();
+						int postCount = dir.List().Length;
+						Assert.IsTrue(postCount < preCount);
+					}
+				}
+				
+				dir.Close();
+			}
+		}
+		
+		/* Test keeping NO commit points.  This is a viable and
+		* useful case eg where you want to build a big index with
+		* autoCommit false and you know there are no readers.
+		*/
+		[Test]
+		public virtual void  TestKeepNoneOnInitDeletionPolicy()
+		{
+			
+			for (int pass = 0; pass < 4; pass++)
+			{
+				
+				bool autoCommit = pass < 2;
+				bool useCompoundFile = (pass % 2) > 0;
+				
+				KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy(this);
+				
+				Directory dir = new RAMDirectory();
+				
+				IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+				writer.SetMaxBufferedDocs(10);
+				writer.SetUseCompoundFile(useCompoundFile);
+				for (int i = 0; i < 107; i++)
+				{
+					AddDoc(writer);
+				}
+				writer.Close();
+				
+				writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+				writer.SetUseCompoundFile(useCompoundFile);
+				writer.Optimize();
+				writer.Close();
+				
+				Assert.AreEqual(2, policy.numOnInit);
+				if (autoCommit)
+				{
+					Assert.IsTrue(policy.numOnCommit > 2);
+				}
+				else
+				{
+					// If we are not auto committing then there should
+					// be exactly 2 commits (one per close above):
+					Assert.AreEqual(2, policy.numOnCommit);
+				}
+				
+				// Simplistic check: just verify the index is in fact
+				// readable:
+				IndexReader reader = IndexReader.Open(dir);
+				reader.Close();
+				
+				dir.Close();
+			}
+		}
+		
+		/*
+		* Test a deletion policy that keeps last N commits.
+		*/
+		[Test]
+		public virtual void  TestKeepLastNDeletionPolicy()
+		{
+			
+			int N = 5;
+			
+			for (int pass = 0; pass < 4; pass++)
+			{
+				
+				bool autoCommit = pass < 2;
+				bool useCompoundFile = (pass % 2) > 0;
+				
+				Directory dir = new RAMDirectory();
+				
+				KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
+				
+				for (int j = 0; j < N + 1; j++)
+				{
+					IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+					writer.SetMaxBufferedDocs(10);
+					writer.SetUseCompoundFile(useCompoundFile);
+					for (int i = 0; i < 17; i++)
+					{
+						AddDoc(writer);
+					}
+					writer.Optimize();
+					writer.Close();
+				}
+				
+				Assert.IsTrue(policy.numDelete > 0);
+				Assert.AreEqual(N + 1, policy.numOnInit);
+				if (autoCommit)
+				{
+					Assert.IsTrue(policy.numOnCommit > 1);
+				}
+				else
+				{
+					Assert.AreEqual(N + 1, policy.numOnCommit);
+				}
+				
+				// Simplistic check: just verify only the past N segments_N's still
+				// exist, and, I can open a reader on each:
+				dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
+				long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
+				for (int i = 0; i < N + 1; i++)
+				{
+					try
+					{
+						IndexReader reader = IndexReader.Open(dir);
+						reader.Close();
+						if (i == N)
+						{
+							Assert.Fail("should have failed on commits prior to last " + N);
+						}
+					}
+					catch (System.IO.IOException e)
+					{
+						if (i != N)
+						{
+							throw e;
+						}
+					}
+					if (i < N)
+					{
+						dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+					}
+					gen--;
+				}
+				
+				dir.Close();
+			}
+		}
+		
+		/*
+		* Test a deletion policy that keeps last N commits
+		* around, with reader doing deletes.
+		*/
+		[Test]
+		public virtual void  TestKeepLastNDeletionPolicyWithReader()
+		{
+			
+			int N = 10;
+			
+			for (int pass = 0; pass < 4; pass++)
+			{
+				
+				bool autoCommit = pass < 2;
+				bool useCompoundFile = (pass % 2) > 0;
+				
+				KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
+				
+				Directory dir = new RAMDirectory();
+				IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+				writer.SetUseCompoundFile(useCompoundFile);
+				writer.Close();
+				Term searchTerm = new Term("content", "aaa");
+				Query query = new TermQuery(searchTerm);
+				
+				for (int i = 0; i < N + 1; i++)
+				{
+					writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+					writer.SetUseCompoundFile(useCompoundFile);
+					for (int j = 0; j < 17; j++)
+					{
+						AddDoc(writer);
+					}
+					// this is a commit when autoCommit=false:
+					writer.Close();
+					IndexReader reader = IndexReader.Open(dir, policy);
+					reader.DeleteDocument(3 * i + 1);
+					reader.SetNorm(4 * i + 1, "content", 2.0F);
+					IndexSearcher searcher = new IndexSearcher(reader);
+					Hits hits = searcher.Search(query);
+					Assert.AreEqual(16 * (1 + i), hits.Length());
+					// this is a commit when autoCommit=false:
+					reader.Close();
+					searcher.Close();
+				}
+				writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+				writer.SetUseCompoundFile(useCompoundFile);
+				writer.Optimize();
+				// this is a commit when autoCommit=false:
+				writer.Close();
+				
+				Assert.AreEqual(2 * (N + 2), policy.numOnInit);
+				if (autoCommit)
+				{
+					Assert.IsTrue(policy.numOnCommit > 2 * (N + 2) - 1);
+				}
+				else
+				{
+					Assert.AreEqual(2 * (N + 2) - 1, policy.numOnCommit);
+				}
+				
+				IndexSearcher searcher2 = new IndexSearcher(dir);
+				Hits hits2 = searcher2.Search(query);
+				Assert.AreEqual(176, hits2.Length());
+				
+				// Simplistic check: just verify only the past N segments_N's still
+				// exist, and, I can open a reader on each:
+				long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
+				
+				dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
+				int expectedCount = 176;
+				
+				for (int i = 0; i < N + 1; i++)
+				{
+					try
+					{
+						IndexReader reader = IndexReader.Open(dir);
+						
+						// Work backwards in commits on what the expected
+						// count should be.  Only check this in the
+						// autoCommit false case:
+						if (!autoCommit)
+						{
+							searcher2 = new IndexSearcher(reader);
+							hits2 = searcher2.Search(query);
+							if (i > 1)
+							{
+								if (i % 2 == 0)
+								{
+									expectedCount += 1;
+								}
+								else
+								{
+									expectedCount -= 17;
+								}
+							}
+							Assert.AreEqual(expectedCount, hits2.Length());
+							searcher2.Close();
+						}
+						reader.Close();
+						if (i == N)
+						{
+							Assert.Fail("should have failed on commits before last 5");
+						}
+					}
+					catch (System.IO.IOException e)
+					{
+						if (i != N)
+						{
+							throw e;
+						}
+					}
+					if (i < N)
+					{
+						dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+					}
+					gen--;
+				}
+				
+				dir.Close();
+			}
+		}
+		
+		/*
+		* Test a deletion policy that keeps last N commits
+		* around, through creates.
+		*/
+		[Test]
+		public virtual void  TestKeepLastNDeletionPolicyWithCreates()
+		{
+			
+			int N = 10;
+			
+			for (int pass = 0; pass < 4; pass++)
+			{
+				
+				bool autoCommit = pass < 2;
+				bool useCompoundFile = (pass % 2) > 0;
+				
+				KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
+				
+				Directory dir = new RAMDirectory();
+				IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+				writer.SetMaxBufferedDocs(10);
+				writer.SetUseCompoundFile(useCompoundFile);
+				writer.Close();
+				Term searchTerm = new Term("content", "aaa");
+				Query query = new TermQuery(searchTerm);
+				
+				for (int i = 0; i < N + 1; i++)
+				{
+					
+					writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+					writer.SetMaxBufferedDocs(10);
+					writer.SetUseCompoundFile(useCompoundFile);
+					for (int j = 0; j < 17; j++)
+					{
+						AddDoc(writer);
+					}
+					// this is a commit when autoCommit=false:
+					writer.Close();
+					IndexReader reader = IndexReader.Open(dir, policy);
+					reader.DeleteDocument(3);
+					reader.SetNorm(5, "content", 2.0F);
+					IndexSearcher searcher = new IndexSearcher(reader);
+					Hits hits = searcher.Search(query);
+					Assert.AreEqual(16, hits.Length());
+					// this is a commit when autoCommit=false:
+					reader.Close();
+					searcher.Close();
+					
+					writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+					// This will not commit: there are no changes
+					// pending because we opened for "create":
+					writer.Close();
+				}
+				
+				Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
+				if (autoCommit)
+				{
+					Assert.IsTrue(policy.numOnCommit > 3 * (N + 1) - 1);
+				}
+				else
+				{
+					Assert.AreEqual(2 * (N + 1), policy.numOnCommit);
+				}
+				
+				IndexSearcher searcher2 = new IndexSearcher(dir);
+				Hits hits2 = searcher2.Search(query);
+				Assert.AreEqual(0, hits2.Length());
+				
+				// Simplistic check: just verify only the past N segments_N's still
+				// exist, and, I can open a reader on each:
+				long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
+				
+				dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
+				int expectedCount = 0;
+				
+				for (int i = 0; i < N + 1; i++)
+				{
+					try
+					{
+						IndexReader reader = IndexReader.Open(dir);
+						
+						// Work backwards in commits on what the expected
+						// count should be.  Only check this in the
+						// autoCommit false case:
+						if (!autoCommit)
+						{
+							searcher2 = new IndexSearcher(reader);
+							hits2 = searcher2.Search(query);
+							Assert.AreEqual(expectedCount, hits2.Length());
+							searcher2.Close();
+							if (expectedCount == 0)
+							{
+								expectedCount = 16;
+							}
+							else if (expectedCount == 16)
+							{
+								expectedCount = 17;
+							}
+							else if (expectedCount == 17)
+							{
+								expectedCount = 0;
+							}
+						}
+						reader.Close();
+						if (i == N)
+						{
+							Assert.Fail("should have failed on commits before last " + N);
+						}
+					}
+					catch (System.IO.IOException e)
+					{
+						if (i != N)
+						{
+							throw e;
+						}
+					}
+					if (i < N)
+					{
+						dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+					}
+					gen--;
+				}
+				
+				dir.Close();
+			}
+		}
+		
+		private void  AddDoc(IndexWriter writer)
+		{
+			Document doc = new Document();
+			doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
+			writer.AddDocument(doc);
+		}
+	}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestDoc.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestDoc.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestDoc.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestDoc.cs Tue Jul 15 14:44:04 2008
@@ -19,25 +19,27 @@
 
 using NUnit.Framework;
 
-using SimpleAnalyzer = Lucene.Net.Analysis.SimpleAnalyzer;
-using Analyzer = Lucene.Net.Analysis.Analyzer;
-using FSDirectory = Lucene.Net.Store.FSDirectory;
-using Directory = Lucene.Net.Store.Directory;
+//using TestRunner = junit.textui.TestRunner;
 using Document = Lucene.Net.Documents.Document;
-using Similarity = Lucene.Net.Search.Similarity;
+using Directory = Lucene.Net.Store.Directory;
+using FSDirectory = Lucene.Net.Store.FSDirectory;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using SimpleAnalyzer = Lucene.Net.Analysis.SimpleAnalyzer;
 using FileDocument = Lucene.Net.Demo.FileDocument;
+using Similarity = Lucene.Net.Search.Similarity;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 
 namespace Lucene.Net.Index
 {
 	
 	
-	/// <summary>JUnit adaptation of an older test case DocTest.</summary>
-	/// <author>  dmitrys@earthlink.net
-	/// </author>
-	/// <version>  $Id: TestDoc.java 150536 2004-09-28 18:15:52Z cutting $
+	/// <summary>JUnit adaptation of an older test case DocTest.
+	/// 
+	/// </summary>
+	/// <version>  $Id: TestDoc.java 583534 2007-10-10 16:46:35Z mikemccand $
 	/// </version>
 	[TestFixture]
-    public class TestDoc
+	public class TestDoc
 	{
 		
 		/// <summary>Main for running test case by itself. </summary>
@@ -57,7 +59,7 @@
 		/// a few text files created in the current working directory.
 		/// </summary>
 		[SetUp]
-        public virtual void  SetUp()
+		public virtual void  SetUp()
 		{
 			workDir = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", "tempDir"), "TestDoc"));
 			System.IO.Directory.CreateDirectory(workDir.FullName);
@@ -131,76 +133,74 @@
 		/// assert various things about the segment.
 		/// </summary>
 		[Test]
-        public virtual void  TestIndexAndMerge()
+		public virtual void  TestIndexAndMerge()
 		{
-            System.IO.MemoryStream sw = new System.IO.MemoryStream();
-            System.IO.StreamWriter out_Renamed = new System.IO.StreamWriter(sw);
+			System.IO.MemoryStream sw = new System.IO.MemoryStream();
+			System.IO.StreamWriter out_Renamed = new System.IO.StreamWriter(sw);
 			
-			Directory directory = FSDirectory.GetDirectory(indexDir, true);
-			directory.Close();
+			Directory directory = FSDirectory.GetDirectory(indexDir);
+			IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true);
 			
-            SegmentInfo si1 = IndexDoc("one", "test.txt");
-            PrintSegment(out_Renamed, si1);
+			SegmentInfo si1 = IndexDoc(writer, "test.txt");
+			PrintSegment(out_Renamed, si1);
 			
-            SegmentInfo si2 = IndexDoc("two", "test2.txt");
-            PrintSegment(out_Renamed, si2);
+			SegmentInfo si2 = IndexDoc(writer, "test2.txt");
+			PrintSegment(out_Renamed, si2);
+			writer.Close();
+			directory.Close();
 			
-            SegmentInfo siMerge = Merge(si1, si2, "merge", false);
-            PrintSegment(out_Renamed, siMerge);
+			SegmentInfo siMerge = Merge(si1, si2, "merge", false);
+			PrintSegment(out_Renamed, siMerge);
 			
-            SegmentInfo siMerge2 = Merge(si1, si2, "merge2", false);
-            PrintSegment(out_Renamed, siMerge2);
+			SegmentInfo siMerge2 = Merge(si1, si2, "merge2", false);
+			PrintSegment(out_Renamed, siMerge2);
 			
-            SegmentInfo siMerge3 = Merge(siMerge, siMerge2, "merge3", false);
-            PrintSegment(out_Renamed, siMerge3);
+			SegmentInfo siMerge3 = Merge(siMerge, siMerge2, "merge3", false);
+			PrintSegment(out_Renamed, siMerge3);
 			
 			out_Renamed.Close();
 			sw.Close();
-            System.String multiFileOutput = System.Text.ASCIIEncoding.ASCII.GetString(sw.ToArray());
-            //System.out.println(multiFileOutput);
+			System.String multiFileOutput = System.Text.ASCIIEncoding.ASCII.GetString(sw.ToArray());
+			//System.out.println(multiFileOutput);
 			
-            sw = new System.IO.MemoryStream();
-            out_Renamed = new System.IO.StreamWriter(sw);
+			sw = new System.IO.MemoryStream();
+			out_Renamed = new System.IO.StreamWriter(sw);
 			
-			directory = FSDirectory.GetDirectory(indexDir, true);
-			directory.Close();
+			directory = FSDirectory.GetDirectory(indexDir);
+			writer = new IndexWriter(directory, new SimpleAnalyzer(), true);
 			
-            si1 = IndexDoc("one", "test.txt");
-            PrintSegment(out_Renamed, si1);
+			si1 = IndexDoc(writer, "test.txt");
+			PrintSegment(out_Renamed, si1);
 			
-            si2 = IndexDoc("two", "test2.txt");
-            PrintSegment(out_Renamed, si2);
+			si2 = IndexDoc(writer, "test2.txt");
+			PrintSegment(out_Renamed, si2);
+			writer.Close();
+			directory.Close();
 			
-            siMerge = Merge(si1, si2, "merge", true);
-            PrintSegment(out_Renamed, siMerge);
+			siMerge = Merge(si1, si2, "merge", true);
+			PrintSegment(out_Renamed, siMerge);
 			
-            siMerge2 = Merge(si1, si2, "merge2", true);
-            PrintSegment(out_Renamed, siMerge2);
+			siMerge2 = Merge(si1, si2, "merge2", true);
+			PrintSegment(out_Renamed, siMerge2);
 			
-            siMerge3 = Merge(siMerge, siMerge2, "merge3", true);
-            PrintSegment(out_Renamed, siMerge3);
+			siMerge3 = Merge(siMerge, siMerge2, "merge3", true);
+			PrintSegment(out_Renamed, siMerge3);
 			
 			out_Renamed.Close();
 			sw.Close();
-            System.String singleFileOutput = System.Text.ASCIIEncoding.ASCII.GetString(sw.ToArray());
+			System.String singleFileOutput = System.Text.ASCIIEncoding.ASCII.GetString(sw.ToArray());
 			
 			Assert.AreEqual(multiFileOutput, singleFileOutput);
 		}
 		
 		
-		private SegmentInfo IndexDoc(System.String segment, System.String fileName)
+		private SegmentInfo IndexDoc(IndexWriter writer, System.String fileName)
 		{
-			Directory directory = FSDirectory.GetDirectory(indexDir, false);
-			Analyzer analyzer = new SimpleAnalyzer();
-			DocumentWriter writer = new DocumentWriter(directory, analyzer, Similarity.GetDefault(), 1000);
-			
-			System.IO.FileInfo file = new System.IO.FileInfo(System.IO.Path.Combine(workDir.FullName, fileName));
-			Lucene.Net.Documents.Document doc = FileDocument.Document(file);
-			
-			writer.AddDocument(segment, doc);
-			
-			directory.Close();
-            return new SegmentInfo(segment, 1, directory, false, false);
+			System.IO.FileInfo file = new System.IO.FileInfo(workDir.FullName + "\\" + fileName);
+			Document doc = FileDocument.Document(file);
+			writer.AddDocument(doc);
+			writer.Flush();
+			return writer.NewestSegment();
 		}
 		
 		
@@ -228,8 +228,8 @@
 			}
 			
 			directory.Close();
-            return new SegmentInfo(merged, si1.docCount + si2.docCount, directory, useCompoundFile, true);
-        }
+			return new SegmentInfo(merged, si1.docCount + si2.docCount, directory, useCompoundFile, true);
+		}
 		
 		
 		private void  PrintSegment(System.IO.StreamWriter out_Renamed, SegmentInfo si)

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestDocumentWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestDocumentWriter.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestDocumentWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestDocumentWriter.cs Tue Jul 15 14:44:04 2008
@@ -19,19 +19,20 @@
 
 using NUnit.Framework;
 
-using Analyzer = Lucene.Net.Analysis.Analyzer;
-using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
-using TokenStream = Lucene.Net.Analysis.TokenStream;
-using WhitespaceTokenizer = Lucene.Net.Analysis.WhitespaceTokenizer;
-using Lucene.Net.Documents;
-using Similarity = Lucene.Net.Search.Similarity;
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using Fieldable = Lucene.Net.Documents.Fieldable;
+using TermVector = Lucene.Net.Documents.Field.TermVector;
 using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using Lucene.Net.Analysis;
+using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
 
 namespace Lucene.Net.Index
 {
 	
 	[TestFixture]
-	public class TestDocumentWriter
+	public class TestDocumentWriter : LuceneTestCase
 	{
 		private class AnonymousClassAnalyzer : Analyzer
 		{
@@ -52,8 +53,7 @@
 				}
 				
 			}
-
-            public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
+			public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
 			{
 				return new WhitespaceTokenizer(reader);
 			}
@@ -63,48 +63,158 @@
 				return 500;
 			}
 		}
-		private RAMDirectory dir;
-		
-        // public TestDocumentWriter(System.String s)
-        // {
-        // }
 		
-        [SetUp]
-        public virtual void  SetUp()
+		private class AnonymousClassAnalyzer1 : Analyzer
 		{
-			dir = new RAMDirectory();
+			public AnonymousClassAnalyzer1(TestDocumentWriter enclosingInstance)
+			{
+				InitBlock(enclosingInstance);
+			}
+			
+			private class AnonymousClassTokenFilter:TokenFilter
+			{
+				private void  InitBlock(AnonymousClassAnalyzer1 enclosingInstance)
+				{
+					this.enclosingInstance = enclosingInstance;
+				}
+				private AnonymousClassAnalyzer1 enclosingInstance;
+				public AnonymousClassAnalyzer1 Enclosing_Instance
+				{
+					get
+					{
+						return enclosingInstance;
+					}
+					
+				}
+				internal AnonymousClassTokenFilter(AnonymousClassAnalyzer1 enclosingInstance, Lucene.Net.Analysis.TokenStream Param1):base(Param1)
+				{
+					InitBlock(enclosingInstance);
+				}
+				internal bool first = true;
+				internal Token buffered;
+				
+				public override Token Next()
+				{
+					return input.Next();
+				}
+				
+				public override Token Next(Token result)
+				{
+					if (buffered != null)
+					{
+						Token t = buffered;
+						buffered = null;
+						return t;
+					}
+					Token t2 = input.Next(result);
+					if (t2 == null)
+						return null;
+					if (System.Char.IsDigit(t2.TermBuffer()[0]))
+					{
+						t2.SetPositionIncrement(t2.TermBuffer()[0] - '0');
+					}
+					if (first)
+					{
+						// set payload on first position only
+						t2.SetPayload(new Payload(new byte[]{100}));
+						first = false;
+					}
+					
+					// index a "synonym" for every token
+					buffered = (Token) t2.Clone();
+					buffered.SetPayload(null);
+					buffered.SetPositionIncrement(0);
+					buffered.SetTermBuffer(new char[]{'b'}, 0, 1);
+					
+					return t2;
+				}
+			}
+			private void  InitBlock(TestDocumentWriter enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private TestDocumentWriter enclosingInstance;
+			public TestDocumentWriter Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
+			{
+				return new AnonymousClassTokenFilter(this, new WhitespaceTokenizer(reader));
+			}
 		}
 		
-		[TearDown]
-        public virtual void  TearDown()
+		private class AnonymousClassTokenStream : TokenStream
 		{
+			public AnonymousClassTokenStream(TestDocumentWriter enclosingInstance)
+			{
+				InitBlock(enclosingInstance);
+			}
+			private void  InitBlock(TestDocumentWriter enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private TestDocumentWriter enclosingInstance;
+			public TestDocumentWriter Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			private System.String[] tokens = new System.String[]{"term1", "term2", "term3", "term2"};
+			private int index = 0;
 			
+			public override Token Next()
+			{
+				if (index == tokens.Length)
+				{
+					return null;
+				}
+				else
+				{
+					return new Token(tokens[index++], 0, 0);
+				}
+			}
+		}
+		private RAMDirectory dir;
+		
+		[SetUp]
+		public override void  SetUp()
+		{
+			dir = new RAMDirectory();
 		}
 		
 		[Test]
-        public virtual void  Test()
+		public virtual void  Test()
 		{
 			Assert.IsTrue(dir != null);
 		}
 		
 		[Test]
-        public virtual void  TestAddDocument()
+		public virtual void  TestAddDocument()
 		{
-			Lucene.Net.Documents.Document testDoc = new Lucene.Net.Documents.Document();
+			Document testDoc = new Document();
 			DocHelper.SetupDoc(testDoc);
 			Analyzer analyzer = new WhitespaceAnalyzer();
-			Lucene.Net.Search.Similarity similarity = Lucene.Net.Search.Similarity.GetDefault();
-			DocumentWriter writer = new DocumentWriter(dir, analyzer, similarity, 50);
-			System.String segName = "test";
-			writer.AddDocument(segName, testDoc);
+			IndexWriter writer = new IndexWriter(dir, analyzer, true);
+			writer.AddDocument(testDoc);
+			writer.Flush();
+			SegmentInfo info = writer.NewestSegment();
+			writer.Close();
 			//After adding the document, we should be able to read it back in
-			SegmentReader reader = SegmentReader.Get(new SegmentInfo(segName, 1, dir));
+			SegmentReader reader = SegmentReader.Get(info);
 			Assert.IsTrue(reader != null);
-			Lucene.Net.Documents.Document doc = reader.Document(0);
+			Document doc = reader.Document(0);
 			Assert.IsTrue(doc != null);
 			
 			//System.out.println("Document: " + doc);
-			Field[] fields = doc.GetFields("textField2");
+			Fieldable[] fields = doc.GetFields("textField2");
 			Assert.IsTrue(fields != null && fields.Length == 1);
 			Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_2_TEXT));
 			Assert.IsTrue(fields[0].IsTermVectorStored());
@@ -126,31 +236,34 @@
 			Assert.IsTrue(fields != null && fields.Length == 1);
 			Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_3_TEXT));
 			
-			// test that the norm file is not present if omitNorms is true
-			for (int i = 0; i < reader.FieldInfos.Size(); i++)
+			// test that the norms are not present in the segment if
+			// omitNorms is true
+			for (int i = 0; i < reader.FieldInfos().Size(); i++)
 			{
-				FieldInfo fi = reader.FieldInfos.FieldInfo(i);
+				FieldInfo fi = reader.FieldInfos().FieldInfo(i);
 				if (fi.IsIndexed())
 				{
-					Assert.IsTrue(fi.omitNorms == !dir.FileExists(segName + ".f" + i));
+					Assert.IsTrue(fi.omitNorms == !reader.HasNorms(fi.Name_ForNUnitTest));
 				}
 			}
 		}
 		
 		[Test]
-        public virtual void  TestPositionIncrementGap()
+		public virtual void  TestPositionIncrementGap()
 		{
 			Analyzer analyzer = new AnonymousClassAnalyzer(this);
 			
-			Lucene.Net.Search.Similarity similarity = Lucene.Net.Search.Similarity.GetDefault();
-			DocumentWriter writer = new DocumentWriter(dir, analyzer, similarity, 50);
-			Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
+			IndexWriter writer = new IndexWriter(dir, analyzer, true);
+			
+			Document doc = new Document();
 			doc.Add(new Field("repeated", "repeated one", Field.Store.YES, Field.Index.TOKENIZED));
 			doc.Add(new Field("repeated", "repeated two", Field.Store.YES, Field.Index.TOKENIZED));
 			
-			System.String segName = "test";
-			writer.AddDocument(segName, doc);
-			SegmentReader reader = SegmentReader.Get(new SegmentInfo(segName, 1, dir));
+			writer.AddDocument(doc);
+			writer.Flush();
+			SegmentInfo info = writer.NewestSegment();
+			writer.Close();
+			SegmentReader reader = SegmentReader.Get(info);
 			
 			TermPositions termPositions = reader.TermPositions(new Term("repeated", "repeated"));
 			Assert.IsTrue(termPositions.Next());
@@ -159,5 +272,95 @@
 			Assert.AreEqual(0, termPositions.NextPosition());
 			Assert.AreEqual(502, termPositions.NextPosition());
 		}
+		
+		[Test]
+		public virtual void  TestTokenReuse()
+		{
+			Analyzer analyzer = new AnonymousClassAnalyzer1(this);
+			
+			IndexWriter writer = new IndexWriter(dir, analyzer, true);
+			
+			Document doc = new Document();
+			doc.Add(new Field("f1", "a 5 a a", Field.Store.YES, Field.Index.TOKENIZED));
+			
+			writer.AddDocument(doc);
+			writer.Flush();
+			SegmentInfo info = writer.NewestSegment();
+			writer.Close();
+			SegmentReader reader = SegmentReader.Get(info);
+			
+			TermPositions termPositions = reader.TermPositions(new Term("f1", "a"));
+			Assert.IsTrue(termPositions.Next());
+			int freq = termPositions.Freq();
+			Assert.AreEqual(3, freq);
+			Assert.AreEqual(0, termPositions.NextPosition());
+			Assert.AreEqual(true, termPositions.IsPayloadAvailable());
+			Assert.AreEqual(6, termPositions.NextPosition());
+			Assert.AreEqual(false, termPositions.IsPayloadAvailable());
+			Assert.AreEqual(7, termPositions.NextPosition());
+			Assert.AreEqual(false, termPositions.IsPayloadAvailable());
+		}
+		
+		
+		[Test]
+		public virtual void  TestPreAnalyzedField()
+		{
+			IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), true);
+			Document doc = new Document();
+			
+			doc.Add(new Field("preanalyzed", new AnonymousClassTokenStream(this), TermVector.NO));
+			
+			writer.AddDocument(doc);
+			writer.Flush();
+			SegmentInfo info = writer.NewestSegment();
+			writer.Close();
+			SegmentReader reader = SegmentReader.Get(info);
+			
+			TermPositions termPositions = reader.TermPositions(new Term("preanalyzed", "term1"));
+			Assert.IsTrue(termPositions.Next());
+			Assert.AreEqual(1, termPositions.Freq());
+			Assert.AreEqual(0, termPositions.NextPosition());
+			
+			termPositions.Seek(new Term("preanalyzed", "term2"));
+			Assert.IsTrue(termPositions.Next());
+			Assert.AreEqual(2, termPositions.Freq());
+			Assert.AreEqual(1, termPositions.NextPosition());
+			Assert.AreEqual(3, termPositions.NextPosition());
+			
+			termPositions.Seek(new Term("preanalyzed", "term3"));
+			Assert.IsTrue(termPositions.Next());
+			Assert.AreEqual(1, termPositions.Freq());
+			Assert.AreEqual(2, termPositions.NextPosition());
+		}
+		
+		/// <summary> Test adding two fields with the same name, but 
+		/// with different term vector setting (LUCENE-766).
+		/// </summary>
+		[Test]
+		public virtual void  TestMixedTermVectorSettingsSameField()
+		{
+			Document doc = new Document();
+			// f1 first without tv then with tv
+			doc.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.UN_TOKENIZED, TermVector.NO));
+			doc.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.UN_TOKENIZED, TermVector.WITH_POSITIONS_OFFSETS));
+			// f2 first with tv then without tv
+			doc.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.UN_TOKENIZED, TermVector.WITH_POSITIONS_OFFSETS));
+			doc.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.UN_TOKENIZED, TermVector.NO));
+			
+			RAMDirectory ram = new RAMDirectory();
+			IndexWriter writer = new IndexWriter(ram, new StandardAnalyzer(), true);
+			writer.AddDocument(doc);
+			writer.Close();
+			
+			IndexReader reader = IndexReader.Open(ram);
+			// f1
+			TermFreqVector tfv1 = reader.GetTermFreqVector(0, "f1");
+			Assert.IsNotNull(tfv1);
+			Assert.AreEqual(2, tfv1.GetTerms().Length, "the 'with_tv' setting should rule!");
+			// f2
+			TermFreqVector tfv2 = reader.GetTermFreqVector(0, "f2");
+			Assert.IsNotNull(tfv2);
+			Assert.AreEqual(2, tfv2.GetTerms().Length, "the 'with_tv' setting should rule!");
+		}
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldInfos.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestFieldInfos.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldInfos.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldInfos.cs Tue Jul 15 14:44:04 2008
@@ -20,8 +20,9 @@
 using NUnit.Framework;
 
 using Document = Lucene.Net.Documents.Document;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
 using IndexOutput = Lucene.Net.Store.IndexOutput;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 
 namespace Lucene.Net.Index
 {
@@ -29,28 +30,23 @@
 	
 	//import org.cnlp.utils.properties.ResourceBundleHelper;
 	[TestFixture]
-	public class TestFieldInfos
+	public class TestFieldInfos : LuceneTestCase
 	{
 		
 		private Lucene.Net.Documents.Document testDoc = new Lucene.Net.Documents.Document();
 		
-        // public TestFieldInfos(System.String s)
-        // {
-        // }
+		// public TestFieldInfos(System.String s)
+		// {
+		// }
 		
-        [SetUp]
-        public virtual void  SetUp()
+		[SetUp]
+		public override void SetUp()
 		{
 			DocHelper.SetupDoc(testDoc);
 		}
 		
-		[TearDown]
-        public virtual void  TearDown()
-		{
-		}
-		
 		[Test]
-        public virtual void  Test()
+		public virtual void  Test()
 		{
 			//Positive test of FieldInfos
 			Assert.IsTrue(testDoc != null);
@@ -93,7 +89,7 @@
 				
 				dir.Close();
 			}
-			catch (System.IO.IOException e)
+			catch (System.IO.IOException)
 			{
 				Assert.IsTrue(false);
 			}



Mime
View raw message