lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject [Lucene.Net] svn commit: r1082321 [1/4] - in /incubator/lucene.net/trunk/C#/src: Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/ Lucene.Net/Index/ Lucene.Net/Search/ Lucene.Net/Store/ Lucene.Net/Util/ Test/Index/ Test/Search/ Test/Util/
Date Wed, 16 Mar 2011 22:14:43 GMT
Author: digy
Date: Wed Mar 16 22:14:41 2011
New Revision: 1082321

URL: http://svn.apache.org/viewvc?rev=1082321&view=rev
Log:
LUCENENET-399 
Core:
Lucene.Net.Analysis + Lucene.Net.Index + Lucene.Net.Search + Lucene.Net.Store + Lucene.Net.Util
Test:
Lucene.Net.Analysis + ucene.Net.Search + Lucene.Net.Util

PS: TestCachingSpanFilter.cs should be added to test project

Added:
    incubator/lucene.net/trunk/C#/src/Test/Index/index.30.cfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.30.nocfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Search/TestCachingSpanFilter.cs
Modified:
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/BaseCharFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PerFieldAnalyzerWrapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteBlockPool.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReusableStringReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriterPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHash.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/CachingSpanFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/CachingWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldCache.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldCacheImpl.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldDoc.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/IndexSearcher.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TimeLimitingCollector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/WildcardTermEnum.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/Directory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/FSDirectory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/FileSwitchDirectory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/IndexOutput.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/MMapDirectory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/NativeFSLockFactory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMFile.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMOutputStream.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/BitVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Constants.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/NumericUtils.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSet.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/PriorityQueue.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestBackwardsCompatibility.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestByteSlices.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestDoc.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexReaderReopen.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/CachingWrapperFilterHelper.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestCachingWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSort.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestWildcard.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/LuceneTestCase.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/TestNumericUtils.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/TestOpenBitSet.cs

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/BaseCharFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/BaseCharFilter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/BaseCharFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/BaseCharFilter.cs Wed Mar 16 22:14:41 2011
@@ -16,88 +16,82 @@
  */
 
 using System;
+using Lucene.Net.Util;
 
 namespace Lucene.Net.Analysis
 {
-	
-	/// <summary> Base utility class for implementing a {@link CharFilter}.
-	/// You subclass this, and then record mappings by calling
-	/// {@link #addOffCorrectMap}, and then invoke the correct
-	/// method to correct an offset.
-	/// 
-	/// <p/><b>NOTE</b>: This class is not particularly efficient.
-	/// For example, a new class instance is created for every
-	/// call to {@link #addOffCorrectMap}, which is then appended
-	/// to a private list.
-	/// </summary>
-	public abstract class BaseCharFilter:CharFilter
-	{
-		
-		//private List<OffCorrectMap> pcmList;
-		private System.Collections.IList pcmList;
-		
-		public BaseCharFilter(CharStream in_Renamed):base(in_Renamed)
-		{
-		}
-		
-		/// <summary>Retrieve the corrected offset.  Note that this method
-		/// is slow, if you correct positions far before the most
-		/// recently added position, as it's a simple linear
-		/// search backwards through all offset corrections added
-		/// by {@link #addOffCorrectMap}.
-		/// </summary>
-		public /*protected internal*/ override int Correct(int currentOff)
-		{
-			if (pcmList == null || (pcmList.Count == 0))
-			{
-				return currentOff;
-			}
-			for (int i = pcmList.Count - 1; i >= 0; i--)
-			{
-				if (currentOff >= ((OffCorrectMap) pcmList[i]).off)
-				{
-					return currentOff + ((OffCorrectMap) pcmList[i]).cumulativeDiff;
-				}
-			}
-			return currentOff;
-		}
-		
-		protected internal virtual int GetLastCumulativeDiff()
-		{
-			return pcmList == null || (pcmList.Count == 0)?0:((OffCorrectMap) pcmList[pcmList.Count - 1]).cumulativeDiff;
-		}
-		
-		protected internal virtual void  AddOffCorrectMap(int off, int cumulativeDiff)
-		{
-			if (pcmList == null)
-			{
-				pcmList = new System.Collections.ArrayList();
-			}
-			pcmList.Add(new OffCorrectMap(off, cumulativeDiff));
-		}
-		
-		internal class OffCorrectMap
-		{
-			
-			internal int off;
-			internal int cumulativeDiff;
-			
-			internal OffCorrectMap(int off, int cumulativeDiff)
-			{
-				this.off = off;
-				this.cumulativeDiff = cumulativeDiff;
-			}
-			
-			public override System.String ToString()
-			{
-				System.Text.StringBuilder sb = new System.Text.StringBuilder();
-				sb.Append('(');
-				sb.Append(off);
-				sb.Append(',');
-				sb.Append(cumulativeDiff);
-				sb.Append(')');
-				return sb.ToString();
-			}
-		}
-	}
+
+    /// <summary>
+    /// * Base utility class for implementing a {@link CharFilter}.
+    /// * You subclass this, and then record mappings by calling
+    /// * {@link #addOffCorrectMap}, and then invoke the correct
+    /// * method to correct an offset.
+    /// </summary>
+    public abstract class BaseCharFilter : CharFilter
+    {
+
+        private int[] offsets;
+        private int[] diffs;
+        private int size = 0;
+
+        public BaseCharFilter(CharStream @in) : base(@in)
+        {
+        }
+
+        /** Retrieve the corrected offset. */
+        //@Override
+        public override int Correct(int currentOff)
+        {
+            if (offsets == null || currentOff < offsets[0])
+            {
+                return currentOff;
+            }
+
+            int hi = size - 1;
+            if (currentOff >= offsets[hi])
+                return currentOff + diffs[hi];
+
+            int lo = 0;
+            int mid = -1;
+
+            while (hi >= lo)
+            {
+                mid = SupportClass.Number.URShift(lo + hi, 1);
+                if (currentOff < offsets[mid])
+                    hi = mid - 1;
+                else if (currentOff > offsets[mid])
+                    lo = mid + 1;
+                else
+                    return currentOff + diffs[mid];
+            }
+
+            if (currentOff < offsets[mid])
+                return mid == 0 ? currentOff : currentOff + diffs[mid - 1];
+            else
+                return currentOff + diffs[mid];
+        }
+
+        protected int GetLastCumulativeDiff()
+        {
+            return offsets == null ?
+              0 : diffs[size - 1];
+        }
+
+        protected void AddOffCorrectMap(int off, int cumulativeDiff)
+        {
+            if (offsets == null)
+            {
+                offsets = new int[64];
+                diffs = new int[64];
+            }
+            else if (size == offsets.Length)
+            {
+                offsets = ArrayUtil.Grow(offsets);
+                diffs = ArrayUtil.Grow(diffs);
+            }
+
+            offsets[size] = off;
+            diffs[size++] = cumulativeDiff;
+        }
+    }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PerFieldAnalyzerWrapper.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/PerFieldAnalyzerWrapper.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PerFieldAnalyzerWrapper.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PerFieldAnalyzerWrapper.cs Wed Mar 16 22:14:41 2011
@@ -127,6 +127,15 @@ namespace Lucene.Net.Analysis
 				analyzer = defaultAnalyzer;
 			return analyzer.GetPositionIncrementGap(fieldName);
 		}
+
+        /// <summary> Return the offsetGap from the analyzer assigned to field </summary>
+        public override int GetOffsetGap(Lucene.Net.Documents.Fieldable field)
+        {
+            Analyzer analyzer = (Analyzer)analyzerMap[field.Name()];
+            if (analyzer == null)
+                analyzer = defaultAnalyzer;
+            return analyzer.GetOffsetGap(field);
+        }
 		
 		public override System.String ToString()
 		{

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs Wed Mar 16 22:14:41 2011
@@ -314,21 +314,11 @@ namespace Lucene.Net.Analysis.Standard
 			return base.Next();
 		}
 		
-		/*
-		* (non-Javadoc)
-		*
-		* @see Lucene.Net.Analysis.TokenStream#reset()
-		*/
-		public override void  Reset()
-		{
-			base.Reset();
-			scanner.Yyreset(input);
-		}
-		
+				
 		public override void  Reset(System.IO.TextReader reader)
 		{
 			base.Reset(reader);
-			Reset();
+			scanner.Reset(reader);
 		}
 		
 		/// <summary> Prior to https://issues.apache.org/jira/browse/LUCENE-1068, StandardTokenizer mischaracterized as acronyms tokens like www.abc.com

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs Wed Mar 16 22:14:41 2011
@@ -258,6 +258,19 @@ namespace Lucene.Net.Analysis.Standard
 		{
 			return yychar;
 		}
+
+        /**
+        * Resets the Tokenizer to a new Reader.
+        */
+        internal void Reset(System.IO.TextReader r)
+        {
+            // reset to default buffer size, if buffer has grown
+            if (zzBuffer.Length > ZZ_BUFFERSIZE)
+            {
+                zzBuffer = new char[ZZ_BUFFERSIZE];
+            }
+            Yyreset(r);
+        }
 		
 		/// <summary> Fills Lucene token with the current token text.</summary>
 		internal void  GetText(Token t)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex Wed Mar 16 22:14:41 2011
@@ -1,145 +1,156 @@
-package org.apache.lucene.analysis.standard;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
-
-NOTE: if you change StandardTokenizerImpl.jflex and need to regenerate
-      the tokenizer, remember to use JRE 1.4 to run jflex (before
-      Lucene 3.0).  This grammar now uses constructs (eg :digit:,
-      :letter:) whose meaning can vary according to the JRE used to
-      run jflex.  See
-      https://issues.apache.org/jira/browse/LUCENE-1126 for details.
-
-*/
-
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-
-%%
-
-%class StandardTokenizerImpl
-%unicode
-%integer
-%function getNextToken
-%pack
-%char
-
-%{
-
-public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
-public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
-public static final int ACRONYM           = StandardTokenizer.ACRONYM;
-public static final int COMPANY           = StandardTokenizer.COMPANY;
-public static final int EMAIL             = StandardTokenizer.EMAIL;
-public static final int HOST              = StandardTokenizer.HOST;
-public static final int NUM               = StandardTokenizer.NUM;
-public static final int CJ                = StandardTokenizer.CJ;
-/**
- * @deprecated this solves a bug where HOSTs that end with '.' are identified
- *             as ACRONYMs. It is deprecated and will be removed in the next
- *             release.
- */
-public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
-
-public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
-
-public final int yychar()
-{
-    return yychar;
-}
-
-/**
- * Fills Lucene token with the current token text.
- */
-final void getText(Token t) {
-  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
-}
-
-/**
- * Fills TermAttribute with the current token text.
- */
-final void getText(TermAttribute t) {
-  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
-}
-
-%}
-
-THAI       = [\u0E00-\u0E59]
-
-// basic word: a sequence of digits & letters (includes Thai to enable ThaiAnalyzer to function)
-ALPHANUM   = ({LETTER}|{THAI}|[:digit:])+
-
-// internal apostrophes: O'Reilly, you're, O'Reilly's
-// use a post-filter to remove possessives
-APOSTROPHE =  {ALPHA} ("'" {ALPHA})+
-
-// acronyms: U.S.A., I.B.M., etc.
-// use a post-filter to remove dots
-ACRONYM    =  {LETTER} "." ({LETTER} ".")+
-
-ACRONYM_DEP	= {ALPHANUM} "." ({ALPHANUM} ".")+
-
-// company names like AT&T and Excite@Home.
-COMPANY    =  {ALPHA} ("&"|"@") {ALPHA}
-
-// email addresses
-EMAIL      =  {ALPHANUM} (("."|"-"|"_") {ALPHANUM})* "@" {ALPHANUM} (("."|"-") {ALPHANUM})+
-
-// hostname
-HOST       =  {ALPHANUM} ((".") {ALPHANUM})+
-
-// floating point, serial, model numbers, ip addresses, etc.
-// every other segment must have at least one digit
-NUM        = ({ALPHANUM} {P} {HAS_DIGIT}
-           | {HAS_DIGIT} {P} {ALPHANUM}
-           | {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
-           | {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
-           | {ALPHANUM} {P} {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
-           | {HAS_DIGIT} {P} {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+)
-
-// punctuation
-P	         = ("_"|"-"|"/"|"."|",")
-
-// at least one digit
-HAS_DIGIT  = ({LETTER}|[:digit:])* [:digit:] ({LETTER}|[:digit:])*
-
-ALPHA      = ({LETTER})+
-
-// From the JFlex manual: "the expression that matches everything of <a> not matched by <b> is !(!<a>|<b>)"
-LETTER     = !(![:letter:]|{CJ})
-
-// Chinese and Japanese (but NOT Korean, which is included in [:letter:])
-CJ         = [\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f]
-
-WHITESPACE = \r\n | [ \r\n\t\f]
-
-%%
-
-{ALPHANUM}                                                     { return ALPHANUM; }
-{APOSTROPHE}                                                   { return APOSTROPHE; }
-{ACRONYM}                                                      { return ACRONYM; }
-{COMPANY}                                                      { return COMPANY; }
-{EMAIL}                                                        { return EMAIL; }
-{HOST}                                                         { return HOST; }
-{NUM}                                                          { return NUM; }
-{CJ}                                                           { return CJ; }
-{ACRONYM_DEP}                                                  { return ACRONYM_DEP; }
-
-/** Ignore the rest */
-. | {WHITESPACE}                                               { /* ignore */ }
+package org.apache.lucene.analysis.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+
+NOTE: if you change StandardTokenizerImpl.jflex and need to regenerate
+      the tokenizer, remember to use JRE 1.4 to run jflex (before
+      Lucene 3.0).  This grammar now uses constructs (eg :digit:,
+      :letter:) whose meaning can vary according to the JRE used to
+      run jflex.  See
+      https://issues.apache.org/jira/browse/LUCENE-1126 for details.
+
+*/
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+%%
+
+%class StandardTokenizerImpl
+%unicode
+%integer
+%function getNextToken
+%pack
+%char
+
+%{
+
+public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
+public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
+public static final int ACRONYM           = StandardTokenizer.ACRONYM;
+public static final int COMPANY           = StandardTokenizer.COMPANY;
+public static final int EMAIL             = StandardTokenizer.EMAIL;
+public static final int HOST              = StandardTokenizer.HOST;
+public static final int NUM               = StandardTokenizer.NUM;
+public static final int CJ                = StandardTokenizer.CJ;
+/**
+ * @deprecated this solves a bug where HOSTs that end with '.' are identified
+ *             as ACRONYMs. It is deprecated and will be removed in the next
+ *             release.
+ */
+public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
+
+public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
+
+public final int yychar()
+{
+    return yychar;
+}
+
+/**
+ * Resets the Tokenizer to a new Reader.
+ */
+final void reset(java.io.Reader r) {
+  // reset to default buffer size, if buffer has grown
+  if (zzBuffer.length > ZZ_BUFFERSIZE) {
+    zzBuffer = new char[ZZ_BUFFERSIZE];
+  }
+  yyreset(r);
+}
+
+/**
+ * Fills Lucene token with the current token text.
+ */
+final void getText(Token t) {
+  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+}
+
+/**
+ * Fills TermAttribute with the current token text.
+ */
+final void getText(TermAttribute t) {
+  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+}
+
+%}
+
+THAI       = [\u0E00-\u0E59]
+
+// basic word: a sequence of digits & letters (includes Thai to enable ThaiAnalyzer to function)
+ALPHANUM   = ({LETTER}|{THAI}|[:digit:])+
+
+// internal apostrophes: O'Reilly, you're, O'Reilly's
+// use a post-filter to remove possessives
+APOSTROPHE =  {ALPHA} ("'" {ALPHA})+
+
+// acronyms: U.S.A., I.B.M., etc.
+// use a post-filter to remove dots
+ACRONYM    =  {LETTER} "." ({LETTER} ".")+
+
+ACRONYM_DEP	= {ALPHANUM} "." ({ALPHANUM} ".")+
+
+// company names like AT&T and Excite@Home.
+COMPANY    =  {ALPHA} ("&"|"@") {ALPHA}
+
+// email addresses
+EMAIL      =  {ALPHANUM} (("."|"-"|"_") {ALPHANUM})* "@" {ALPHANUM} (("."|"-") {ALPHANUM})+
+
+// hostname
+HOST       =  {ALPHANUM} ((".") {ALPHANUM})+
+
+// floating point, serial, model numbers, ip addresses, etc.
+// every other segment must have at least one digit
+NUM        = ({ALPHANUM} {P} {HAS_DIGIT}
+           | {HAS_DIGIT} {P} {ALPHANUM}
+           | {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
+           | {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+           | {ALPHANUM} {P} {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+           | {HAS_DIGIT} {P} {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+)
+
+// punctuation
+P	         = ("_"|"-"|"/"|"."|",")
+
+// at least one digit
+HAS_DIGIT  = ({LETTER}|[:digit:])* [:digit:] ({LETTER}|[:digit:])*
+
+ALPHA      = ({LETTER})+
+
+// From the JFlex manual: "the expression that matches everything of <a> not matched by <b> is !(!<a>|<b>)"
+LETTER     = !(![:letter:]|{CJ})
+
+// Chinese and Japanese (but NOT Korean, which is included in [:letter:])
+CJ         = [\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f]
+
+WHITESPACE = \r\n | [ \r\n\t\f]
+
+%%
+
+{ALPHANUM}                                                     { return ALPHANUM; }
+{APOSTROPHE}                                                   { return APOSTROPHE; }
+{ACRONYM}                                                      { return ACRONYM; }
+{COMPANY}                                                      { return COMPANY; }
+{EMAIL}                                                        { return EMAIL; }
+{HOST}                                                         { return HOST; }
+{NUM}                                                          { return NUM; }
+{CJ}                                                           { return CJ; }
+{ACRONYM_DEP}                                                  { return ACRONYM_DEP; }
+
+/** Ignore the rest */
+. | {WHITESPACE}                                               { /* ignore */ }

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteBlockPool.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ByteBlockPool.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteBlockPool.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteBlockPool.cs Wed Mar 16 22:14:41 2011
@@ -33,6 +33,7 @@
 * hit a non-zero byte. */
 
 using System;
+using System.Collections.Generic;
 
 namespace Lucene.Net.Index
 {
@@ -47,6 +48,8 @@ namespace Lucene.Net.Index
 		public /*internal*/ abstract class Allocator
 		{
 			public /*internal*/ abstract void  RecycleByteBlocks(byte[][] blocks, int start, int end);
+            //abstract void recycleByteBlocks(List blocks); DIGY
+            public /*internal*/ abstract void RecycleByteBlocks(System.Collections.ArrayList blocks);
 			public /*internal*/ abstract byte[] GetByteBlock(bool trackAllocations);
 		}
 		

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CheckIndex.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs Wed Mar 16 22:14:41 2011
@@ -668,8 +668,11 @@ namespace Lucene.Net.Index
 				while (it.MoveNext())
 				{
 					System.String fieldName = (System.String) it.Current;
-					reader.Norms(fieldName, b, 0);
-					++status.totFields;
+                    if (reader.HasNorms(fieldName))
+                    {
+                        reader.Norms(fieldName, b, 0);
+                        ++status.totFields;
+                    }
 				}
 				
 				Msg("OK [" + status.totFields + " fields]");

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs Wed Mar 16 22:14:41 2011
@@ -40,7 +40,7 @@ namespace Lucene.Net.Index
 		protected internal System.Collections.IList mergeThreads = new System.Collections.ArrayList();
 		
 		// Max number of threads allowed to be merging at once
-		private int maxThreadCount = 3;
+		private int maxThreadCount = 1;
 		
 		protected internal Directory dir;
 		
@@ -173,16 +173,30 @@ namespace Lucene.Net.Index
 		
 		private int MergeThreadCount()
 		{
-			lock (this)
-			{
-				int count = 0;
-				int numThreads = mergeThreads.Count;
-				for (int i = 0; i < numThreads; i++)
-					if (((MergeThread) mergeThreads[i]).IsAlive)
-						count++;
-				return count;
-			}
+            return MergeThreadCount(false);
 		}
+
+        private int MergeThreadCount(bool excludeDone)
+        {
+            lock (this)
+            {
+                int count = 0;
+                int numThreads = mergeThreads.Count;
+                for (int i = 0; i < numThreads; i++)
+                {
+                    MergeThread t = (MergeThread)mergeThreads[i];
+                    if (t.IsAlive)
+                    {
+                        MergePolicy.OneMerge runningMerge = t.GetRunningMerge();
+                        if (!excludeDone || (runningMerge != null && !runningMerge.mergeDone))
+                        {
+                            count++;
+                        }
+                    }
+                }
+                return count;
+            }
+        }
 		
 		public override void  Merge(IndexWriter writer)
 		{
@@ -236,7 +250,7 @@ namespace Lucene.Net.Index
 					lock (this)
 					{
 						MergeThread merger;
-						while (MergeThreadCount() >= maxThreadCount)
+						while (MergeThreadCount(true) >= maxThreadCount)
 						{
 							if (Verbose())
 								Message("    too many merge threads running; stalling...");
@@ -256,8 +270,7 @@ namespace Lucene.Net.Index
 						if (Verbose())
 							Message("  consider merge " + merge.SegString(dir));
 						
-						System.Diagnostics.Debug.Assert(MergeThreadCount() < maxThreadCount);
-						
+												
 						// OK to spawn a new merge thread to handle this
 						// merge:
 						merger = GetMergeThread(writer, merge);

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DirectoryReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryReader.cs Wed Mar 16 22:14:41 2011
@@ -98,14 +98,18 @@ namespace Lucene.Net.Index
 		private int termInfosIndexDivisor;
 		
 		private bool rollbackHasChanges;
-		private SegmentInfos rollbackSegmentInfos;
-		
+				
 		private SegmentReader[] subReaders;
 		private int[] starts; // 1st docno for each segment
 		private System.Collections.IDictionary normsCache = new System.Collections.Hashtable();
 		private int maxDoc = 0;
 		private int numDocs = - 1;
 		private bool hasDeletions = false;
+        
+        // Max version in index as of when we opened; this can be
+        // > our current segmentInfos version in case we were
+        // opened on a past IndexCommit:
+        private long maxIndexVersion;
 		
 		internal static IndexReader Open(Directory directory, IndexDeletionPolicy deletionPolicy, IndexCommit commit, bool readOnly, int termInfosIndexDivisor)
 		{
@@ -170,7 +174,7 @@ namespace Lucene.Net.Index
 		{
 			this.directory = writer.GetDirectory();
 			this.readOnly = true;
-			this.segmentInfos = infos;
+			segmentInfos = infos;
 			segmentInfosStart = (SegmentInfos) infos.Clone();
 			this.termInfosIndexDivisor = termInfosIndexDivisor;
 			if (!readOnly)
@@ -193,7 +197,7 @@ namespace Lucene.Net.Index
 				bool success = false;
 				try
 				{
-					SegmentInfo info = infos.Info(upto);
+					SegmentInfo info = infos.Info(i);
 					if (info.dir == dir)
 					{
 						readers[upto++] = writer.readerPool.GetReadOnlyClone(info, true, termInfosIndexDivisor);
@@ -407,6 +411,11 @@ namespace Lucene.Net.Index
 					hasDeletions = true;
 			}
 			starts[subReaders.Length] = maxDoc;
+
+            if (!readOnly)
+            {
+                maxIndexVersion = SegmentInfos.ReadCurrentVersion(directory);
+            }
 		}
 		
 		public override System.Object Clone()
@@ -451,139 +460,163 @@ namespace Lucene.Net.Index
 		
 		public override IndexReader Reopen()
 		{
-			lock (this)
-			{
-				// Preserve current readOnly
-				return DoReopen(readOnly, null);
-			}
+	        // Preserve current readOnly
+			return DoReopen(readOnly, null);
 		}
 		
 		public override IndexReader Reopen(bool openReadOnly)
 		{
-			lock (this)
-			{
-				return DoReopen(openReadOnly, null);
-			}
+			return DoReopen(openReadOnly, null);
 		}
 		
 		public override IndexReader Reopen(IndexCommit commit)
 		{
-			lock (this)
-			{
-				return DoReopen(true, commit);
-			}
-		}
-		
-		private IndexReader DoReopen(bool openReadOnly, IndexCommit commit)
-		{
-			lock (this)
-			{
-				EnsureOpen();
-				
-				System.Diagnostics.Debug.Assert(commit == null || openReadOnly);
-				
-				// If we were obtained by writer.getReader(), re-ask the
-				// writer to get a new reader.
-				if (writer != null)
-				{
-					System.Diagnostics.Debug.Assert(readOnly);
-					
-					if (!openReadOnly)
-					{
-						throw new System.ArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
-					}
-					
-					if (commit != null)
-					{
-						throw new System.ArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
-					}
-					
-					if (!writer.IsOpen(true))
-					{
-						throw new AlreadyClosedException("cannot reopen: the IndexWriter this reader was obtained from is now closed");
-					}
-					
-					// TODO: right now we *always* make a new reader; in
-					// the future we could have write make some effort to
-					// detect that no changes have occurred
-					IndexReader reader = writer.GetReader();
-					reader.SetDisableFakeNorms(GetDisableFakeNorms());
-					return reader;
-				}
-				
-				if (commit == null)
-				{
-					if (hasChanges)
-					{
-						// We have changes, which means we are not readOnly:
-						System.Diagnostics.Debug.Assert(readOnly == false);
-						// and we hold the write lock:
-						System.Diagnostics.Debug.Assert(writeLock != null);
-						// so no other writer holds the write lock, which
-						// means no changes could have been done to the index:
-						System.Diagnostics.Debug.Assert(IsCurrent());
-						
-						if (openReadOnly)
-						{
-							return (IndexReader) Clone(openReadOnly);
-						}
-						else
-						{
-							return this;
-						}
-					}
-					else if (IsCurrent())
-					{
-						if (openReadOnly != readOnly)
-						{
-							// Just fallback to clone
-							return (IndexReader) Clone(openReadOnly);
-						}
-						else
-						{
-							return this;
-						}
-					}
-				}
-				else
-				{
-					if (directory != commit.GetDirectory())
-						throw new System.IO.IOException("the specified commit does not match the specified Directory");
-					if (segmentInfos != null && commit.GetSegmentsFileName().Equals(segmentInfos.GetCurrentSegmentFileName()))
-					{
-						if (readOnly != openReadOnly)
-						{
-							// Just fallback to clone
-							return (IndexReader) Clone(openReadOnly);
-						}
-						else
-						{
-							return this;
-						}
-					}
-				}
-				
-				return (IndexReader) new AnonymousClassFindSegmentsFile1(openReadOnly, this, directory).Run(commit);
-			}
-		}
-		
-		private DirectoryReader DoReopen(SegmentInfos infos, bool doClone, bool openReadOnly)
-		{
-			lock (this)
-			{
-				DirectoryReader reader;
-				if (openReadOnly)
-				{
-					reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
-				}
-				else
-				{
-					reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
-				}
-				reader.SetDisableFakeNorms(GetDisableFakeNorms());
-				return reader;
-			}
+			return DoReopen(true, commit);
 		}
+
+        private IndexReader DoReopenFromWriter(bool openReadOnly, IndexCommit commit)
+        {
+            System.Diagnostics.Debug.Assert(readOnly);
+
+            if (!openReadOnly)
+            {
+                throw new System.ArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
+            }
+
+            if (commit != null)
+            {
+                throw new System.ArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
+            }
+
+            // TODO: right now we *always* make a new reader; in
+            // the future we could have write make some effort to
+            // detect that no changes have occurred
+            return writer.GetReader();
+        }
+
+        private IndexReader DoReopen(bool openReadOnly, IndexCommit commit)
+        {
+            EnsureOpen();
+
+            System.Diagnostics.Debug.Assert(commit == null || openReadOnly);
+
+            // If we were obtained by writer.getReader(), re-ask the
+            // writer to get a new reader.
+            if (writer != null)
+            {
+                return DoReopenFromWriter(openReadOnly, commit);
+            }
+            else
+            {
+                return DoReopenNoWriter(openReadOnly, commit);
+            }
+        }
+                
+        private IndexReader DoReopenNoWriter(bool openReadOnly, IndexCommit commit)
+        {
+            lock (this)
+            {
+                if (commit == null)
+                {
+                    if (hasChanges)
+                    {
+                        // We have changes, which means we are not readOnly:
+                        System.Diagnostics.Debug.Assert(readOnly == false);
+                        // and we hold the write lock:
+                        System.Diagnostics.Debug.Assert(writeLock != null);
+                        // so no other writer holds the write lock, which
+                        // means no changes could have been done to the index:
+                        System.Diagnostics.Debug.Assert(IsCurrent());
+
+                        if (openReadOnly)
+                        {
+                            return (IndexReader)Clone(openReadOnly);
+                        }
+                        else
+                        {
+                            return this;
+                        }
+                    }
+                    else if (IsCurrent())
+                    {
+                        if (openReadOnly != readOnly)
+                        {
+                            // Just fallback to clone
+                            return (IndexReader)Clone(openReadOnly);
+                        }
+                        else
+                        {
+                            return this;
+                        }
+                    }
+                }
+                else
+                {
+                    if (directory != commit.GetDirectory())
+                        throw new System.IO.IOException("the specified commit does not match the specified Directory");
+                    if (segmentInfos != null && commit.GetSegmentsFileName().Equals(segmentInfos.GetCurrentSegmentFileName()))
+                    {
+                        if (readOnly != openReadOnly)
+                        {
+                            // Just fallback to clone
+                            return (IndexReader)Clone(openReadOnly);
+                        }
+                        else
+                        {
+                            return this;
+                        }
+                    }
+                }
+
+                return (IndexReader)new AnonymousFindSegmentsFile(directory, openReadOnly, this).Run();
+                //DIGY
+                //return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
+                //  protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
+                //    SegmentInfos infos = new SegmentInfos();
+                //    infos.read(directory, segmentFileName);
+                //    return doReopen(infos, false, openReadOnly);
+                //  }
+                //}.run(commit);
+            }
+        }
+
+        class AnonymousFindSegmentsFile : SegmentInfos.FindSegmentsFile
+        {
+            DirectoryReader enclosingInstance;
+            bool openReadOnly;
+            public AnonymousFindSegmentsFile(Directory dir,bool openReadOnly,DirectoryReader dirReader) : base(dir)
+            {
+                this.openReadOnly = openReadOnly;
+                enclosingInstance = dirReader;
+            }
+
+            public override object DoBody(string segmentFileName)
+            {
+                SegmentInfos infos = new SegmentInfos();
+                infos.Read(directory, segmentFileName);
+                return enclosingInstance.DoReopen(infos, false, openReadOnly);
+            }
+        }
+
+        private DirectoryReader DoReopen(SegmentInfos infos, bool doClone, bool openReadOnly)
+        {
+            lock (this)
+            {
+                DirectoryReader reader;
+                if (openReadOnly)
+                {
+                    reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+                }
+                else
+                {
+                    reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+                }
+                reader.SetDisableFakeNorms(GetDisableFakeNorms());
+                return reader;
+            }
+        }
+
+
 		
 		/// <summary>Version number when this IndexReader was opened. </summary>
 		public override long GetVersion()
@@ -632,19 +665,18 @@ namespace Lucene.Net.Index
 		
 		public override int NumDocs()
 		{
-			lock (this)
+			// Don't call ensureOpen() here (it could affect performance)
+            // NOTE: multiple threads may wind up init'ing
+            // numDocs... but that's harmless
+			if (numDocs == - 1)
 			{
-				// Don't call ensureOpen() here (it could affect performance)
-				if (numDocs == - 1)
-				{
-					// check cache
-					int n = 0; // cache miss--recompute
-					for (int i = 0; i < subReaders.Length; i++)
-						n += subReaders[i].NumDocs(); // sum from readers
-					numDocs = n;
-				}
-				return numDocs;
+				// check cache
+				int n = 0; // cache miss--recompute
+				for (int i = 0; i < subReaders.Length; i++)
+					n += subReaders[i].NumDocs(); // sum from readers
+				numDocs = n;
 			}
+			return numDocs;
 		}
 		
 		public override int MaxDoc()
@@ -871,9 +903,10 @@ namespace Lucene.Net.Index
 					}
 					this.writeLock = writeLock;
 					
-					// we have to check whether index has changed since this reader was opened.
-					// if so, this reader is no longer valid for deletion
-					if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
+                    // we have to check whether index has changed since this reader was opened.
+                    // if so, this reader is no longer valid for
+                    // deletion
+                    if (SegmentInfos.ReadCurrentVersion(directory) > maxIndexVersion)
 					{
 						stale = true;
 						this.writeLock.Release();
@@ -906,8 +939,10 @@ namespace Lucene.Net.Index
 				segmentInfos.SetUserData(commitUserData);
 				// Default deleter (for backwards compatibility) is
 				// KeepOnlyLastCommitDeleter:
-				IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, null, null);
-				
+				IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, null, null, synced);
+
+                segmentInfos.UpdateGeneration(deleter.GetLastSegmentInfos());
+
 				// Checkpoint the state we are about to change, in
 				// case we have to roll back:
 				StartCommit();
@@ -956,6 +991,8 @@ namespace Lucene.Net.Index
 				// files due to this commit:
 				deleter.Checkpoint(segmentInfos, true);
 				deleter.Close();
+
+                maxIndexVersion = segmentInfos.GetVersion();
 				
 				if (writeLock != null)
 				{
@@ -969,7 +1006,6 @@ namespace Lucene.Net.Index
 		internal virtual void  StartCommit()
 		{
 			rollbackHasChanges = hasChanges;
-			rollbackSegmentInfos = (SegmentInfos) segmentInfos.Clone();
 			for (int i = 0; i < subReaders.Length; i++)
 			{
 				subReaders[i].StartCommit();
@@ -978,20 +1014,11 @@ namespace Lucene.Net.Index
 		
 		internal virtual void  RollbackCommit()
 		{
-			hasChanges = rollbackHasChanges;
-			for (int i = 0; i < segmentInfos.Count; i++)
-			{
-				// Rollback each segmentInfo.  Because the
-				// SegmentReader holds a reference to the
-				// SegmentInfo we can't [easily] just replace
-				// segmentInfos, so we reset it in place instead:
-				segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
-			}
-			rollbackSegmentInfos = null;
-			for (int i = 0; i < subReaders.Length; i++)
-			{
-				subReaders[i].RollbackCommit();
-			}
+            hasChanges = rollbackHasChanges;
+            for (int i = 0; i < subReaders.Length; i++)
+            {
+                subReaders[i].RollbackCommit();
+            }
 		}
 
         public override System.Collections.Generic.IDictionary<string, string> GetCommitUserData()
@@ -1033,6 +1060,12 @@ namespace Lucene.Net.Index
 							ioe = e;
 					}
 				}
+
+                // NOTE: only needed in case someone had asked for
+                // FieldCache for top-level reader (which is generally
+                // not a good idea):
+                Lucene.Net.Search.FieldCache_Fields.DEFAULT.Purge(this);
+
 				// throw the first exception
 				if (ioe != null)
 					throw ioe;
@@ -1163,7 +1196,11 @@ namespace Lucene.Net.Index
 				generation = infos.GetGeneration();
 				isOptimized = infos.Count == 1 && !infos.Info(0).HasDeletions();
 			}
-			
+            public override string ToString()
+            {
+                return "DirectoryReader.ReaderCommit(" + segmentsFileName + ")";
+            }
+
 			public override bool IsOptimized()
 			{
 				return isOptimized;
@@ -1203,6 +1240,11 @@ namespace Lucene.Net.Index
 			{
 				return userData;
 			}
+
+            public override void Delete()
+            {
+                throw new System.NotSupportedException("This IndexCommit does not support deletions");
+            }
 		}
 		
 		internal class MultiTermEnum:TermEnum

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocumentsWriter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs Wed Mar 16 22:14:41 2011
@@ -19,16 +19,15 @@ using System;
 
 using Analyzer = Lucene.Net.Analysis.Analyzer;
 using Document = Lucene.Net.Documents.Document;
+using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+using Directory = Lucene.Net.Store.Directory;
+using ArrayUtil = Lucene.Net.Util.ArrayUtil;
+using Constants = Lucene.Net.Util.Constants;
 using IndexSearcher = Lucene.Net.Search.IndexSearcher;
 using Query = Lucene.Net.Search.Query;
 using Scorer = Lucene.Net.Search.Scorer;
 using Similarity = Lucene.Net.Search.Similarity;
 using Weight = Lucene.Net.Search.Weight;
-using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
-using Directory = Lucene.Net.Store.Directory;
-using RAMFile = Lucene.Net.Store.RAMFile;
-using ArrayUtil = Lucene.Net.Util.ArrayUtil;
-using Constants = Lucene.Net.Util.Constants;
 
 namespace Lucene.Net.Index
 {
@@ -105,7 +104,7 @@ namespace Lucene.Net.Index
 		{
 			
 			internal override DocConsumer GetChain(DocumentsWriter documentsWriter)
-            {
+			{
 				/*
 				This is the current indexing chain:
 				
@@ -146,8 +145,8 @@ namespace Lucene.Net.Index
 			freeLevel = (long) (IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB * 1024 * 1024 * 0.95);
 			maxBufferedDocs = IndexWriter.DEFAULT_MAX_BUFFERED_DOCS;
 			skipDocWriter = new SkipDocWriter();
-			byteBlockAllocator = new ByteBlockAllocator(this, BYTE_BLOCK_SIZE);
-            perDocAllocator = new ByteBlockAllocator(this, PER_DOC_BLOCK_SIZE);
+            byteBlockAllocator = new ByteBlockAllocator(this, DocumentsWriter.BYTE_BLOCK_SIZE);
+            perDocAllocator = new ByteBlockAllocator(this,DocumentsWriter.PER_DOC_BLOCK_SIZE);
 			waitQueue = new WaitQueue(this);
 		}
 		
@@ -202,6 +201,14 @@ namespace Lucene.Net.Index
 			{
 				return docWriter.writer.TestPoint(name);
 			}
+
+            public void Clear()
+            {
+                // don't hold onto doc nor analyzer, in case it is
+                // largish:
+                doc = null;
+                analyzer = null;
+            }
 		}
 		
 		/// <summary>Consumer returns this on each doc.  This holds any
@@ -222,50 +229,46 @@ namespace Lucene.Net.Index
 			}
 		}
 		
-        //Create and return a new DocWriterBuffer.
-        internal PerDocBuffer newPerDocBuffer() 
+        /**
+        * Create and return a new DocWriterBuffer.
+        */
+        internal PerDocBuffer NewPerDocBuffer()
         {
-            return new PerDocBuffer(perDocAllocator);
+            return new PerDocBuffer(this);
         }
 
-        /// <summary>RAMFile buffer for DocWriters.</summary>
-        internal class PerDocBuffer:RAMFile 
+        /**
+        * RAMFile buffer for DocWriters.
+        */
+        internal class PerDocBuffer : Lucene.Net.Store.RAMFile
         {
-            public PerDocBuffer(ByteBlockAllocator perDocAllocator)
-			{
-				InitBlock(perDocAllocator);
-			}
-            private void InitBlock(ByteBlockAllocator perDocAllocator)
-			{
-                this.perDocAllocator = perDocAllocator;
-			}
-            private ByteBlockAllocator perDocAllocator;
-
-            /// <summary>
-            ///  Allocate bytes used from shared pool.
-            /// </summary>
-            /// <param name="size">Size of new buffer.  Fixed at <see cref="PER_DOC_BLOCK_SIZE"/>.</param>
-            /// <returns></returns>
-            protected internal byte[] newBuffer(int size) 
+            DocumentsWriter enclosingInstance;
+            public PerDocBuffer(DocumentsWriter enclosingInstance)
+            {
+                this.enclosingInstance = enclosingInstance;
+            }
+            /**
+            * Allocate bytes used from shared pool.
+            */
+            protected byte[] newBuffer(int size)
             {
                 System.Diagnostics.Debug.Assert(size == PER_DOC_BLOCK_SIZE);
-                return perDocAllocator.GetByteBlock(false);
+                return enclosingInstance.perDocAllocator.GetByteBlock(false);
             }
-    
-            //Recycle the bytes used.
-            internal void recycle() 
+
+            /**
+            * Recycle the bytes used.
+            */
+            internal void Recycle()
             {
-                lock(this)
+                lock (this)
                 {
                     if (buffers.Count > 0)
                     {
                         SetLength(0);
 
                         // Recycle the blocks
-                        int blockCount = buffers.Count;
-                        byte[][] blocks = new byte[blockCount][];
-                        buffers.CopyTo(blocks);
-                        perDocAllocator.RecycleByteBlocks(blocks, 0, blockCount);
+                        enclosingInstance.perDocAllocator.RecycleByteBlocks(buffers);
                         buffers.Clear();
                         sizeInBytes = 0;
 
@@ -541,7 +544,7 @@ namespace Lucene.Net.Index
 		internal void  Message(System.String message)
 		{
 			if (infoStream != null)
-                writer.Message("DW: " + message);
+				writer.Message("DW: " + message);
 		}
 
         internal System.Collections.Generic.IList<string> openFiles = new System.Collections.Generic.List<string>();
@@ -635,7 +638,7 @@ namespace Lucene.Net.Index
 						}
 						
 						deletesInRAM.Clear();
-						
+                        deletesFlushed.Clear();
 						openFiles.Clear();
 						
 						for (int i = 0; i < threadStates.Length; i++)
@@ -671,6 +674,10 @@ namespace Lucene.Net.Index
 				{
 					aborting = false;
 					System.Threading.Monitor.PulseAll(this);
+                    if (infoStream != null)
+                    {
+                        Message("docWriter: done abort; abortedFiles=" + abortedFiles);
+                    }
 				}
 			}
 		}
@@ -823,6 +830,12 @@ namespace Lucene.Net.Index
 				return flushState.numDocs;
 			}
 		}
+
+        //ICollection or IDictionary? DIGY
+        internal System.Collections.ICollection GetFlushedFiles()
+        {
+            return flushState.flushedFiles;
+        }
 		
 		/// <summary>Build compound file for the segment we just flushed </summary>
 		internal void  CreateCompoundFile(System.String segment)
@@ -1033,8 +1046,15 @@ namespace Lucene.Net.Index
 			{
 				// This call is not synchronized and does all the
 				// work
-				DocWriter perDoc = state.consumer.ProcessDocument();
-				
+				DocWriter perDoc;
+                try
+                {
+                    perDoc = state.consumer.ProcessDocument();
+                }
+                finally
+                {
+                    docState.Clear();
+                }
 				// This call is synchronized but fast
 				FinishDocument(state, perDoc);
 				success = true;
@@ -1587,12 +1607,12 @@ namespace Lucene.Net.Index
 		{
             public ByteBlockAllocator(DocumentsWriter enclosingInstance, int blockSize)
 			{
-				InitBlock(enclosingInstance, blockSize);
+                this.blockSize = blockSize;
+				InitBlock(enclosingInstance);
 			}
-            private void InitBlock(DocumentsWriter enclosingInstance, int blockSize)
+			private void  InitBlock(DocumentsWriter enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
-                this.blockSize = blockSize;
 			}
 			private DocumentsWriter enclosingInstance;
 			public DocumentsWriter Enclosing_Instance
@@ -1601,12 +1621,12 @@ namespace Lucene.Net.Index
 				{
 					return enclosingInstance;
 				}
+				
 			}
 
             int blockSize;
-
 			internal System.Collections.ArrayList freeByteBlocks = new System.Collections.ArrayList();
-
+            
 			/* Allocate another byte[] from the shared pool */
 			public /*internal*/ override byte[] GetByteBlock(bool trackAllocations)
 			{
@@ -1623,7 +1643,7 @@ namespace Lucene.Net.Index
 						// vectors) and things that do (freq/prox
 						// postings).
                         Enclosing_Instance.numBytesAlloc += blockSize;
-                        b = new byte[blockSize];
+						b = new byte[blockSize];
 					}
 					else
 					{
@@ -1633,7 +1653,7 @@ namespace Lucene.Net.Index
 						b = (byte[]) tempObject;
 					}
 					if (trackAllocations)
-                        Enclosing_Instance.numBytesUsed += blockSize;
+						Enclosing_Instance.numBytesUsed += blockSize;
 					System.Diagnostics.Debug.Assert(Enclosing_Instance.numBytesUsed <= Enclosing_Instance.numBytesAlloc);
 					return b;
 				}
@@ -1644,10 +1664,27 @@ namespace Lucene.Net.Index
 			{
 				lock (Enclosing_Instance)
 				{
-					for (int i = start; i < end; i++)
-						freeByteBlocks.Add(blocks[i]);
+                    for (int i = start; i < end; i++)
+                    {
+                        freeByteBlocks.Add(blocks[i]);
+                        blocks[i] = null;
+                    }
+                    if (enclosingInstance.infoStream != null && blockSize != 1024)
+                    {
+                        enclosingInstance.Message("DW.recycleByteBlocks blockSize=" + blockSize + " count=" + (end - start) + " total now " + freeByteBlocks.Count);
+                    }
 				}
 			}
+
+            public /*internal*/ override void RecycleByteBlocks(System.Collections.ArrayList blocks)
+            {
+                lock (Enclosing_Instance)
+                {
+                    int size = blocks.Count;
+                    for(int i=0;i<size;i++)
+                        freeByteBlocks.Add(blocks[i]);
+                }
+            }
 		}
 		
 		/* Initial chunks size of the shared int[] blocks used to
@@ -1695,7 +1732,6 @@ namespace Lucene.Net.Index
 			lock (this)
 			{
 				numBytesAlloc += numBytes;
-				System.Diagnostics.Debug.Assert(numBytesUsed <= numBytesAlloc);
 			}
 		}
 		
@@ -1716,17 +1752,21 @@ namespace Lucene.Net.Index
                 for (int i = start; i < end; i++)
                 {
                     freeIntBlocks.Add(blocks[i]);
+                    blocks[i] = null;
+                }
+                if (infoStream != null)
+                {
+                    Message("DW.recycleIntBlocks count=" + (end - start) + " total now " + freeIntBlocks.Count);
                 }
 			}
 		}
 		
 		internal ByteBlockAllocator byteBlockAllocator;
 
-        internal const int PER_DOC_BLOCK_SIZE = 1024;
-
-        internal ByteBlockAllocator perDocAllocator;
-
+        internal static int PER_DOC_BLOCK_SIZE = 1024;
 
+        ByteBlockAllocator perDocAllocator;
+		
 		/* Initial chunk size of the shared char[] blocks used to
 		store term text */
 		internal const int CHAR_BLOCK_SHIFT = 14;
@@ -1771,10 +1811,15 @@ namespace Lucene.Net.Index
 		{
 			lock (this)
 			{
-				for (int i = 0; i < numBlocks; i++)
-				{
-					freeCharBlocks.Add(blocks[i]);
-				}
+                for (int i = 0; i < numBlocks; i++)
+                {
+                    freeCharBlocks.Add(blocks[i]);
+                    blocks[i] = null;
+                }
+                if (infoStream != null)
+                {
+                    Message("DW.recycleCharBlocks count=" + numBlocks + " total now " + freeCharBlocks.Count);
+                }
 			}
 		}
 		
@@ -1783,19 +1828,20 @@ namespace Lucene.Net.Index
 			return System.String.Format(nf, "{0:f}", new System.Object[] { (v / 1024F / 1024F) });
 		}
 
+
         /* We have four pools of RAM: Postings, byte blocks
-         * (holds freq/prox posting data), char blocks (holds
-         * characters in the term) and per-doc buffers (stored fields/term vectors).  
-         * Different docs require varying amount of storage from 
-         * these four classes.
-         * 
-         * For example, docs with many unique single-occurrence
-         * short terms will use up the Postings RAM and hardly any
-         * of the other two.  Whereas docs with very large terms
-         * will use alot of char blocks RAM and relatively less of
-         * the other two.  This method just frees allocations from
-         * the pools once we are over-budget, which balances the
-         * pools to match the current docs. */
+        * (holds freq/prox posting data), char blocks (holds
+        * characters in the term) and per-doc buffers (stored fields/term vectors).  
+        * Different docs require varying amount of storage from 
+        * these four classes.
+        * 
+        * For example, docs with many unique single-occurrence
+        * short terms will use up the Postings RAM and hardly any
+        * of the other two.  Whereas docs with very large terms
+        * will use alot of char blocks RAM and relatively less of
+        * the other two.  This method just frees allocations from
+        * the pools once we are over-budget, which balances the
+        * pools to match the current docs. */
 		internal void  BalanceRAM()
 		{
 			
@@ -1808,13 +1854,14 @@ namespace Lucene.Net.Index
 			{
 				
 				if (infoStream != null)
-                    Message("  RAM: now balance allocations: usedMB=" + ToMB(numBytesUsed) + 
+					Message(
+                        "  RAM: now balance allocations: usedMB=" + ToMB(numBytesUsed) + 
                         " vs trigger=" + ToMB(flushTrigger) + 
                         " allocMB=" + ToMB(numBytesAlloc) + 
                         " deletesMB=" + ToMB(deletesRAMUsed) + 
                         " vs trigger=" + ToMB(freeTrigger) + 
-                        " byteBlockFree=" + ToMB(byteBlockAllocator.freeByteBlocks.Count * BYTE_BLOCK_SIZE) + 
-                        " perDocFree=" + ToMB(perDocAllocator.freeByteBlocks.Count * PER_DOC_BLOCK_SIZE) + 
+                        " byteBlockFree=" + ToMB(byteBlockAllocator.freeByteBlocks.Count * BYTE_BLOCK_SIZE) +
+                        " perDocFree=" + ToMB(perDocAllocator.freeByteBlocks.Count * PER_DOC_BLOCK_SIZE) +
                         " charBlockFree=" + ToMB(freeCharBlocks.Count * CHAR_BLOCK_SIZE * CHAR_NUM_BYTE));
 				
 				long startBytesAlloc = numBytesAlloc + deletesRAMUsed;
@@ -1832,17 +1879,17 @@ namespace Lucene.Net.Index
 					
 					lock (this)
 					{
-                        if (0 == perDocAllocator.freeByteBlocks.Count 
-                            && 0 == byteBlockAllocator.freeByteBlocks.Count 
-                            && 0 == freeCharBlocks.Count 
-                            && 0 == freeIntBlocks.Count
-                            && !any) 
+                        if (0 == perDocAllocator.freeByteBlocks.Count
+                              && 0 == byteBlockAllocator.freeByteBlocks.Count
+                              && 0 == freeCharBlocks.Count
+                              && 0 == freeIntBlocks.Count
+                              && !any)
 						{
 							// Nothing else to free -- must flush now.
 							bufferIsFull = numBytesUsed + deletesRAMUsed > flushTrigger;
 							if (infoStream != null)
 							{
-								if (numBytesUsed > flushTrigger)
+                                if (bufferIsFull)
 									Message("    nothing to free; now set bufferIsFull");
 								else
 									Message("    nothing to free");
@@ -1868,21 +1915,20 @@ namespace Lucene.Net.Index
 							freeIntBlocks.RemoveAt(freeIntBlocks.Count - 1);
 							numBytesAlloc -= INT_BLOCK_SIZE * INT_NUM_BYTE;
 						}
-                        
-                        if ((3 == iter % 5) && perDocAllocator.freeByteBlocks.Count > 0) 
+
+                        if ((3 == iter % 5) && perDocAllocator.freeByteBlocks.Count > 0)
                         {
                             // Remove upwards of 32 blocks (each block is 1K)
-                            for (int i = 0; i < 32; ++i) 
+                            for (int i = 0; i < 32; ++i)
                             {
-                                perDocAllocator.freeByteBlocks.RemoveAt (perDocAllocator.freeByteBlocks.Count - 1);
+                                perDocAllocator.freeByteBlocks.Remove(perDocAllocator.freeByteBlocks.Count - 1);
                                 numBytesAlloc -= PER_DOC_BLOCK_SIZE;
-                                if (perDocAllocator.freeByteBlocks.Count == 0) 
+                                if (perDocAllocator.freeByteBlocks.Count == 0)
                                 {
                                     break;
                                 }
                             }
                         }
-        
 					}
 					
 					if ((4 == iter % 5) && any)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/FieldsReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs Wed Mar 16 22:14:41 2011
@@ -117,7 +117,9 @@ namespace Lucene.Net.Index
 				else
 					format = firstInt;
 				
-				if (format > FieldsWriter.FORMAT_CURRENT)
+				if (format > FieldsWriter.FORMAT_CURRENT
+                    /* extra support for Lucene 3.0 indexes: */ && format != FieldsWriter.FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS
+                    )
 					throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower");
 				
 				if (format > FieldsWriter.FORMAT)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/FieldsWriter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs Wed Mar 16 22:14:41 2011
@@ -39,6 +39,12 @@ namespace Lucene.Net.Index
 		
 		// Changed strings to UTF8
 		internal const int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = 1;
+                 
+        // Lucene 3.0: Removal of compressed fields: This is only to provide compatibility with 3.0-created indexes
+        // new segments always use the FORMAT_CURRENT. As the index format did not change in 3.0, only
+        // new stored field files that no longer support compression are marked as such to optimize merging.
+        // But 2.9 can still read them.
+        internal static int FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS = 2;
 		
 		// NOTE: if you introduce a new format, make it 1 higher
 		// than the current one, and always change this if you

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/FilterIndexReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs Wed Mar 16 22:14:41 2011
@@ -298,6 +298,10 @@ namespace Lucene.Net.Index
 		protected internal override void  DoClose()
 		{
 			in_Renamed.Close();
+            // NOTE: only needed in case someone had asked for
+            // FieldCache for top-level reader (which is generally
+            // not a good idea):
+            Lucene.Net.Search.FieldCache_Fields.DEFAULT.Purge(this);
 		}
 
 
@@ -335,5 +339,25 @@ namespace Lucene.Net.Index
             System.Diagnostics.Debug.Fail("Port issue:", "Lets see if we need this FilterIndexReader.Clone()"); // {{Aroush-2.9}}
 			return null;
 		}
+
+        /// <summary>
+        /// If the subclass of FilteredIndexReader modifies the
+        /// contents of the FieldCache, you must override this
+        /// method to provide a different key */
+        ///</summary>
+        public override object GetFieldCacheKey() 
+        {
+            return in_Renamed.GetFieldCacheKey();
+        }
+
+        /// <summary>
+        /// If the subclass of FilteredIndexReader modifies the
+        /// deleted docs, you must override this method to provide
+        /// a different key */
+        /// </summary>
+        public override object GetDeletesCacheKey() 
+        {
+            return in_Renamed.GetDeletesCacheKey();
+        }
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexCommit.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs Wed Mar 16 22:14:41 2011
@@ -65,23 +65,14 @@ namespace Lucene.Net.Index
 		/// and therefore this should only be called by its {@link IndexDeletionPolicy#onInit onInit()} or 
 		/// {@link IndexDeletionPolicy#onCommit onCommit()} methods.
 		/// </summary>
-		public virtual void  Delete()
-		{
-			throw new System.NotSupportedException("This IndexCommit does not support this method.");
-		}
-		
-		public virtual bool IsDeleted()
-		{
-			throw new System.NotSupportedException("This IndexCommit does not support this method.");
-		}
+        public abstract void Delete();
+
+        public abstract bool IsDeleted();
 		
 		/// <summary> Returns true if this commit is an optimized index.</summary>
-		public virtual bool IsOptimized()
-		{
-			throw new System.NotSupportedException("This IndexCommit does not support this method.");
-		}
-		
-		/// <summary> Two IndexCommits are equal if both their Directory and versions are equal.</summary>
+        public abstract bool IsOptimized();
+
+        /// <summary> Two IndexCommits are equal if both their Directory and versions are equal.</summary>
 		public  override bool Equals(System.Object other)
 		{
 			if (other is IndexCommit)
@@ -95,25 +86,19 @@ namespace Lucene.Net.Index
 		
 		public override int GetHashCode()
 		{
-			return GetDirectory().GetHashCode() + GetSegmentsFileName().GetHashCode();
+			return (int)(GetDirectory().GetHashCode() + GetVersion());
 		}
 		
 		/// <summary>Returns the version for this IndexCommit.  This is the
 		/// same value that {@link IndexReader#getVersion} would
 		/// return if it were opened on this commit. 
 		/// </summary>
-		public virtual long GetVersion()
-		{
-			throw new System.NotSupportedException("This IndexCommit does not support this method.");
-		}
+        public abstract long GetVersion();
 		
 		/// <summary>Returns the generation (the _N in segments_N) for this
 		/// IndexCommit 
 		/// </summary>
-		public virtual long GetGeneration()
-		{
-			throw new System.NotSupportedException("This IndexCommit does not support this method.");
-		}
+        public abstract long GetGeneration();
 		
 		/// <summary>Convenience method that returns the last modified time
 		/// of the segments_N file corresponding to this index
@@ -129,9 +114,6 @@ namespace Lucene.Net.Index
 		/// IndexWriter#Commit(Map)} for this commit.  Map is
 		/// String -> String. 
 		/// </summary>
-        public virtual System.Collections.Generic.IDictionary<string, string> GetUserData()
-		{
-			throw new System.NotSupportedException("This IndexCommit does not support this method.");
-		}
+        public abstract System.Collections.Generic.IDictionary<string, string> GetUserData();
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexFileDeleter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs Wed Mar 16 22:14:41 2011
@@ -99,6 +99,9 @@ namespace Lucene.Net.Index
 		private DocumentsWriter docWriter;
 		
 		internal bool startingCommitDeleted;
+        private SegmentInfos lastSegmentInfos;
+
+        private System.Collections.Generic.Dictionary<string, string> synced;
 		
 		/// <summary>Change to true to see details of reference counts when
 		/// infoStream != null 
@@ -116,7 +119,7 @@ namespace Lucene.Net.Index
 		
 		private void  Message(System.String message)
 		{
-			infoStream.WriteLine("IFD [" + SupportClass.ThreadClass.Current().Name + "]: " + message);
+            infoStream.WriteLine("IFD [" + new DateTime().ToString() + "; " + SupportClass.ThreadClass.Current().Name + "]: " + message);
 		}
 		
 		/// <summary> Initialize the deleter: find all previous commits in
@@ -126,11 +129,12 @@ namespace Lucene.Net.Index
 		/// </summary>
 		/// <throws>  CorruptIndexException if the index is corrupt </throws>
 		/// <throws>  IOException if there is a low-level IO error </throws>
-		public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter)
+        public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, System.Collections.Generic.Dictionary<string, string> synced)
 		{
 			
 			this.docWriter = docWriter;
 			this.infoStream = infoStream;
+            this.synced = synced;
 			
 			if (infoStream != null)
 			{
@@ -166,42 +170,58 @@ namespace Lucene.Net.Index
 						// This is a commit (segments or segments_N), and
 						// it's valid (<= the max gen).  Load it, then
 						// incref all files it refers to:
-						if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
-						{
-							if (infoStream != null)
-							{
-								Message("init: load commit \"" + fileName + "\"");
-							}
-							SegmentInfos sis = new SegmentInfos();
-							try
-							{
-								sis.Read(directory, fileName);
-							}
-							catch (System.IO.FileNotFoundException e)
-							{
-								// LUCENE-948: on NFS (and maybe others), if
-								// you have writers switching back and forth
-								// between machines, it's very likely that the
-								// dir listing will be stale and will claim a
-								// file segments_X exists when in fact it
-								// doesn't.  So, we catch this and handle it
-								// as if the file does not exist
-								if (infoStream != null)
-								{
-									Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
-								}
-								sis = null;
-							}
-							if (sis != null)
-							{
-								CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
-								if (sis.GetGeneration() == segmentInfos.GetGeneration())
-								{
-									currentCommitPoint = commitPoint;
-								}
-								commits.Add(commitPoint);
-								IncRef(sis, true);
-							}
+                        if (infoStream != null)
+                        {
+                            Message("init: load commit \"" + fileName + "\"");
+                        }
+                        SegmentInfos sis = new SegmentInfos();
+                        try
+                        {
+                            sis.Read(directory, fileName);
+                        }
+                        catch (System.IO.FileNotFoundException e)
+                        {
+                            // LUCENE-948: on NFS (and maybe others), if
+                            // you have writers switching back and forth
+                            // between machines, it's very likely that the
+                            // dir listing will be stale and will claim a
+                            // file segments_X exists when in fact it
+                            // doesn't.  So, we catch this and handle it
+                            // as if the file does not exist
+                            if (infoStream != null)
+                            {
+                                Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
+                            }
+                            sis = null;
+                        }
+                        catch (System.IO.IOException e)
+                        {
+                            if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
+                            {
+                                throw e;
+                            }
+                            else
+                            {
+                                // Most likely we are opening an index that
+                                // has an aborted "future" commit, so suppress
+                                // exc in this case
+                                sis = null;
+                            }
+                        }
+                        if (sis != null)
+                        {
+                            CommitPoint commitPoint = new CommitPoint(this,commitsToDelete, directory, sis);
+                            if (sis.GetGeneration() == segmentInfos.GetGeneration())
+                            {
+                                currentCommitPoint = commitPoint;
+                            }
+                            commits.Add(commitPoint);
+                            IncRef(sis, true);
+
+                            if (lastSegmentInfos == null || sis.GetGeneration() > lastSegmentInfos.GetGeneration())
+                            {
+                                lastSegmentInfos = sis;
+                            }
 						}
 					}
 				}
@@ -265,6 +285,11 @@ namespace Lucene.Net.Index
 			
 			DeleteCommits();
 		}
+
+        public SegmentInfos GetLastSegmentInfos()
+        {
+            return lastSegmentInfos;
+        }
 		
 		/// <summary> Remove the CommitPoints in the commitsToDelete List by
 		/// DecRef'ing all files from each SegmentInfos.
@@ -545,6 +570,13 @@ namespace Lucene.Net.Index
 				// commit points nor by the in-memory SegmentInfos:
 				DeleteFile(fileName);
 				refCounts.Remove(fileName);
+
+                if (synced != null) {
+                    lock(synced) 
+                    {
+                      synced.Remove(fileName);
+                    }
+                }
 			}
 		}
 		
@@ -556,6 +588,18 @@ namespace Lucene.Net.Index
 				DecRef(it.Current);
 			}
 		}
+
+        public bool Exists(String fileName)
+        {
+            if (!refCounts.ContainsKey(fileName))
+            {
+                return false;
+            }
+            else
+            {
+                return GetRefCount(fileName).count > 0;
+            }
+        }
 		
 		private RefCount GetRefCount(System.String fileName)
 		{
@@ -588,8 +632,14 @@ namespace Lucene.Net.Index
 			while (it.MoveNext())
 			{
 				System.String fileName = (System.String) it.Current;
-				if (!refCounts.ContainsKey(fileName))
-					DeleteFile(fileName);
+                if (!refCounts.ContainsKey(fileName))
+                {
+                    if (infoStream != null)
+                    {
+                        Message("delete new file \"" + fileName + "\"");
+                    }
+                    DeleteFile(fileName);
+                }
 			}
 		}
 		
@@ -711,7 +761,12 @@ namespace Lucene.Net.Index
 				
 				System.Diagnostics.Debug.Assert(!segmentInfos.HasExternalSegments(directory));
 			}
-			
+
+            public override string ToString()
+            {
+                return "IndexFileDeleter.CommitPoint(" + segmentsFileName + ")";
+            }
+
 			public override bool IsOptimized()
 			{
 				return isOptimized;

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs Wed Mar 16 22:14:41 2011
@@ -1774,6 +1774,14 @@ namespace Lucene.Net.Index
 		{
 			return this;
 		}
+
+        /** Expert.  Warning: this returns null if the reader has
+          *  no deletions 
+          */
+        public virtual object GetDeletesCacheKey()
+        {
+            return this;
+        }
 		
 		/// <summary>Returns the number of unique terms (across all fields)
 		/// in this reader.



Mime
View raw message