lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r924791 [1/2] - in /lucene/java/branches/flex_1458: ./ backwards/src/ backwards/src/java/org/apache/lucene/search/ backwards/src/test/org/apache/lucene/analysis/ backwards/src/test/org/apache/lucene/document/ backwards/src/test/org/apache/l...
Date Thu, 18 Mar 2010 14:13:02 GMT
Author: uschindler
Date: Thu Mar 18 14:13:01 2010
New Revision: 924791

URL: http://svn.apache.org/viewvc?rev=924791&view=rev
Log:
LUCENE-2326: merge to flex

Added:
    lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip
      - copied unchanged from r924781, lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java   (with props)
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java   (with props)
Modified:
    lucene/java/branches/flex_1458/   (props changed)
    lucene/java/branches/flex_1458/backwards/src/   (props changed)
    lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java   (props changed)
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java   (props changed)
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestDateTools.java   (props changed)
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestNumberTools.java   (props changed)
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java   (props changed)
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java   (props changed)
    lucene/java/branches/flex_1458/build.xml   (contents, props changed)
    lucene/java/branches/flex_1458/contrib/   (props changed)
    lucene/java/branches/flex_1458/contrib/CHANGES.txt   (props changed)
    lucene/java/branches/flex_1458/contrib/analyzers/common/build.xml
    lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/   (props changed)
    lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
    lucene/java/branches/flex_1458/contrib/highlighter/src/test/   (props changed)
    lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java   (props changed)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Token.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java   (props changed)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttribute.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocInverterPerThread.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java   (props changed)
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java   (props changed)
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java   (props changed)
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java   (props changed)
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java   (props changed)
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java   (contents, props changed)

Propchange: lucene/java/branches/flex_1458/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,4 +2,4 @@
 /lucene/java/branches/lucene_2_9:817269-818600,829134,829881,831036,896850,909334
 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
 /lucene/java/branches/lucene_3_0:880793,896906
-/lucene/java/trunk:824912-825292,827043-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk:824912-825292,827043-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/backwards/src/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -3,5 +3,5 @@
 /lucene/java/branches/lucene_2_9_back_compat_tests/src:818601-821336
 /lucene/java/branches/lucene_3_0/src:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080
 /lucene/java/branches/lucene_3_0_back_compat_tests/src:880869-912395
-/lucene/java/trunk/backwards/src:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214

Propchange: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -3,5 +3,5 @@
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336
 /lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
 /lucene/java/branches/lucene_3_0_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:880869-912395
-/lucene/java/trunk/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:881213,881315,881466,881819,881984,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,5 +2,5 @@
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
 /lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880869-912395
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,5 +2,5 @@
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
 /lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/document/TestDateTools.java:880869-912395
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestDateTools.java:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestDateTools.java:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,5 +2,5 @@
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
 /lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/document/TestNumberTools.java:880869-912395
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/document/TestNumberTools.java:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,5 +2,5 @@
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
 /lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880869-912395
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,885214,912319,912324,912328,912335,912372

Propchange: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -3,5 +3,5 @@
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java:880754,880793,880823,881216,881317,881376,881473,881549,881820,882467,882890,883076,883080,912383
 /lucene/java/branches/lucene_3_0_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:880869-912395
-/lucene/java/trunk/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:922624,922806,923238,924286,924321-924731
+/lucene/java/trunk/backwards/src/test/org/apache/lucene/util/TestAttributeSource.java:922624,922806,923238,924286,924321-924731,924781
 /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882807,882888,882977,883074-883075,883079,885214,912319,912324,912328,912335,912372

Modified: lucene/java/branches/flex_1458/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/build.xml?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/build.xml (original)
+++ lucene/java/branches/flex_1458/build.xml Thu Mar 18 14:13:01 2010
@@ -54,7 +54,7 @@
 
   <patternset id="src.dist.patterns"
               includes="src/,build.xml,*build*.xml,docs/,*.txt,contrib/,*pom.xml*,lib/"
-              excludes="contrib/db/*/lib/,contrib/*/ext-libs/,src/site/build/,contrib/benchmark/temp/,contrib/benchmark/work/,contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/data/"
+              excludes="contrib/db/*/lib/,contrib/*/ext-libs/,src/site/build/,contrib/benchmark/temp/,contrib/benchmark/work/"
   />
   <patternset id="binary.build.dist.patterns"
               includes="${final.name}.jar,${demo.war.name}.war,${demo.name}.jar,docs/,contrib/*/*.jar,contrib/*/*.war, contrib/*/*/*.jar"

Propchange: lucene/java/branches/flex_1458/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,2 +1,2 @@
 /lucene/java/branches/lucene_2_9/build.xml:909334
-/lucene/java/trunk/build.xml:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/build.xml:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,4 +2,4 @@
 /lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
 /lucene/java/branches/lucene_3_0/contrib:880793,896906
-/lucene/java/trunk/contrib:824912-825292,827043-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/contrib:824912-825292,827043-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,4 +2,4 @@
 /lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/CHANGES.txt:880793,896906
-/lucene/java/trunk/contrib/CHANGES.txt:829439-833960,880727-886190,889185,889622,889667,889866-899001,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/contrib/CHANGES.txt:829439-833960,880727-886190,889185,889622,889667,889866-899001,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/build.xml?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/build.xml (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/build.xml Thu Mar 18 14:13:01 2010
@@ -35,20 +35,4 @@
     <path refid="junit-path"/>
     <pathelement location="${build.dir}/classes/java"/>
   </path>	
-
-  <target name="compile-test" depends="download-snowball-vocab-tests, common.compile-test" />
-  <property name="snowball.vocab.rev" value="500"/>
-  <property name="snowball.vocab.url" 
-            value="svn://svn.tartarus.org/snowball/trunk/data"/>
-  <property name="snowball.vocab.dir" value="src/test/org/apache/lucene/analysis/snowball"/>
-		
-  <target name="download-snowball-vocab-tests" depends="compile-core"
-	      description="Downloads Snowball vocabulary tests">
-	<sequential>
-	  <mkdir dir="${snowball.vocab.dir}"/>
-	    <exec dir="${snowball.vocab.dir}" executable="${svn.exe}" failifexecutionfails="false" failonerror="true">
-	      <arg line="checkout --trust-server-cert --non-interactive -r ${snowball.vocab.rev} ${snowball.vocab.url}"/>
-	    </exec>
-	</sequential>
-  </target>
 </project>

Propchange: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/
            ('svn:ignore' removed)

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java Thu Mar 18 14:13:01 2010
@@ -18,12 +18,11 @@ package org.apache.lucene.analysis.snowb
  */
 
 import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.StringReader;
+import java.util.zip.ZipFile;
 
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.KeywordTokenizer;
@@ -35,24 +34,25 @@ import org.apache.lucene.analysis.Tokeni
  */
 public class TestSnowballVocab extends BaseTokenStreamTestCase {
   private Tokenizer tokenizer = new KeywordTokenizer(new StringReader(""));
-  File dataRoot = null;
+  ZipFile zipFile = null;
   
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    try {
-      dataRoot = getDataFile("data");
-    } catch (IOException ioe) {
-      dataRoot = null;
-      System.err.println("WARN: This test was disabled, as the svn checkout of snowball test files is not supported on your system!");
-    }
+    this.zipFile = new ZipFile(getDataFile("TestSnowballVocabData.zip"));
   }
   
+  @Override
+  protected void tearDown() throws Exception {
+    this.zipFile.close();
+    this.zipFile = null;
+    super.tearDown();
+  }
+
   /**
    * Run all languages against their snowball vocabulary tests.
    */
   public void testStemmers() throws IOException {
-    if (dataRoot == null) return;
     assertCorrectOutput("Danish", "danish");
     assertCorrectOutput("Dutch", "dutch");
     assertCorrectOutput("English", "english");
@@ -86,14 +86,12 @@ public class TestSnowballVocab extends B
       throws IOException {
     if (VERBOSE) System.out.println("checking snowball language: " + snowballLanguage);
     TokenStream filter = new SnowballFilter(tokenizer, snowballLanguage);
-    InputStream vocFile = new FileInputStream(new File(dataRoot, 
-        dataDirectory + "/voc.txt"));
-    InputStream outputFile = new FileInputStream(new File(dataRoot, 
-        dataDirectory + "/output.txt"));
+    InputStream voc = zipFile.getInputStream(zipFile.getEntry(dataDirectory + "/voc.txt"));
+    InputStream out = zipFile.getInputStream(zipFile.getEntry(dataDirectory + "/output.txt"));
     BufferedReader vocReader = new BufferedReader(new InputStreamReader(
-        vocFile, "UTF-8"));
+        voc, "UTF-8"));
     BufferedReader outputReader = new BufferedReader(new InputStreamReader(
-        outputFile, "UTF-8"));
+        out, "UTF-8"));
     String inputWord = null;
     while ((inputWord = vocReader.readLine()) != null) {
       String expectedWord = outputReader.readLine();

Propchange: lucene/java/branches/flex_1458/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -2,4 +2,4 @@
 /lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/highlighter/src/test:880793,896906
-/lucene/java/trunk/contrib/highlighter/src/test:829439-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/contrib/highlighter/src/test:829439-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,2 +1,2 @@
 /lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334
-/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Token.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Token.java Thu Mar 18 14:13:01 2010
@@ -64,14 +64,14 @@ import org.apache.lucene.util.AttributeI
   implementing the {@link TokenStream#incrementToken()} API.
   Failing that, to create a new Token you should first use
   one of the constructors that starts with null text.  To load
-  the token from a char[] use {@link #setTermBuffer(char[], int, int)}.
-  To load from a String use {@link #setTermBuffer(String)} or {@link #setTermBuffer(String, int, int)}.
-  Alternatively you can get the Token's termBuffer by calling either {@link #termBuffer()},
+  the token from a char[] use {@link #copyBuffer(char[], int, int)}.
+  To load from a String use {@link #setEmpty} followed by {@link #append(CharSequence)} or {@link #append(CharSequence, int, int)}.
+  Alternatively you can get the Token's termBuffer by calling either {@link #buffer()},
   if you know that your text is shorter than the capacity of the termBuffer
-  or {@link #resizeTermBuffer(int)}, if there is any possibility
+  or {@link #resizeBuffer(int)}, if there is any possibility
   that you may need to grow the buffer. Fill in the characters of your term into this
   buffer, with {@link String#getChars(int, int, char[], int)} if loading from a string,
-  or with {@link System#arraycopy(Object, int, Object, int, int)}, and finally call {@link #setTermLength(int)} to
+  or with {@link System#arraycopy(Object, int, Object, int, int)}, and finally call {@link #setLength(int)} to
   set the length of the term text.  See <a target="_top"
   href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
   for details.</p>
@@ -100,7 +100,7 @@ import org.apache.lucene.util.AttributeI
   </li>
   <li> Copying from one one Token to another (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
   <pre>
-    return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]);
+    return reusableToken.reinit(source.buffer(), 0, source.length(), source.startOffset(), source.endOffset()[, source.type()]);
   </pre>
   </li>
   </ul>
@@ -172,7 +172,7 @@ public class Token extends TermAttribute
    *  @param end end offset
    */
   public Token(String text, int start, int end) {
-    setTermBuffer(text);
+    append(text);
     startOffset = start;
     endOffset = end;
   }
@@ -187,7 +187,7 @@ public class Token extends TermAttribute
    *  @param typ token type
    */
   public Token(String text, int start, int end, String typ) {
-    setTermBuffer(text);
+    append(text);
     startOffset = start;
     endOffset = end;
     type = typ;
@@ -204,7 +204,7 @@ public class Token extends TermAttribute
    * @param flags token type bits
    */
   public Token(String text, int start, int end, int flags) {
-    setTermBuffer(text);
+    append(text);
     startOffset = start;
     endOffset = end;
     this.flags = flags;
@@ -221,7 +221,7 @@ public class Token extends TermAttribute
    * @param end
    */
   public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end) {
-    setTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
+    copyBuffer(startTermBuffer, termBufferOffset, termBufferLength);
     startOffset = start;
     endOffset = end;
   }
@@ -270,7 +270,7 @@ public class Token extends TermAttribute
     corresponding to this token in the source text.
 
     Note that the difference between endOffset() and startOffset() may not be
-    equal to {@link #termLength}, as the term text may have been altered by a
+    equal to {@link #length}, as the term text may have been altered by a
     stemmer or some other filter. */
   public final int startOffset() {
     return startOffset;
@@ -351,7 +351,7 @@ public class Token extends TermAttribute
   @Override
   public String toString() {
     final StringBuilder sb = new StringBuilder();
-    sb.append('(').append(term()).append(',')
+    sb.append('(').append(super.toString()).append(',')
       .append(startOffset).append(',').append(endOffset);
     if (!"word".equals(type))
       sb.append(",type=").append(type);
@@ -387,7 +387,7 @@ public class Token extends TermAttribute
   /** Makes a clone, but replaces the term buffer &
    * start/end offset in the process.  This is more
    * efficient than doing a full clone (and then calling
-   * setTermBuffer) because it saves a wasted copy of the old
+   * {@link #copyBuffer}) because it saves a wasted copy of the old
    * termBuffer. */
   public Token clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
     final Token t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset);
@@ -442,16 +442,16 @@ public class Token extends TermAttribute
   }
 
   /** Shorthand for calling {@link #clear},
-   *  {@link #setTermBuffer(char[], int, int)},
+   *  {@link #copyBuffer(char[], int, int)},
    *  {@link #setStartOffset},
    *  {@link #setEndOffset},
    *  {@link #setType}
    *  @return this Token instance */
   public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
     clearNoTermBuffer();
+    copyBuffer(newTermBuffer, newTermOffset, newTermLength);
     payload = null;
     positionIncrement = 1;
-    setTermBuffer(newTermBuffer, newTermOffset, newTermLength);
     startOffset = newStartOffset;
     endOffset = newEndOffset;
     type = newType;
@@ -459,14 +459,14 @@ public class Token extends TermAttribute
   }
 
   /** Shorthand for calling {@link #clear},
-   *  {@link #setTermBuffer(char[], int, int)},
+   *  {@link #copyBuffer(char[], int, int)},
    *  {@link #setStartOffset},
    *  {@link #setEndOffset}
    *  {@link #setType} on Token.DEFAULT_TYPE
    *  @return this Token instance */
   public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
     clearNoTermBuffer();
-    setTermBuffer(newTermBuffer, newTermOffset, newTermLength);
+    copyBuffer(newTermBuffer, newTermOffset, newTermLength);
     startOffset = newStartOffset;
     endOffset = newEndOffset;
     type = DEFAULT_TYPE;
@@ -474,14 +474,14 @@ public class Token extends TermAttribute
   }
 
   /** Shorthand for calling {@link #clear},
-   *  {@link #setTermBuffer(String)},
+   *  {@link #append(CharSequence)},
    *  {@link #setStartOffset},
    *  {@link #setEndOffset}
    *  {@link #setType}
    *  @return this Token instance */
   public Token reinit(String newTerm, int newStartOffset, int newEndOffset, String newType) {
-    clearNoTermBuffer();
-    setTermBuffer(newTerm);
+    clear();
+    append(newTerm);
     startOffset = newStartOffset;
     endOffset = newEndOffset;
     type = newType;
@@ -489,14 +489,14 @@ public class Token extends TermAttribute
   }
 
   /** Shorthand for calling {@link #clear},
-   *  {@link #setTermBuffer(String, int, int)},
+   *  {@link #append(CharSequence, int, int)},
    *  {@link #setStartOffset},
    *  {@link #setEndOffset}
    *  {@link #setType}
    *  @return this Token instance */
   public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
-    clearNoTermBuffer();
-    setTermBuffer(newTerm, newTermOffset, newTermLength);
+    clear();
+    append(newTerm, newTermOffset, newTermOffset + newTermLength);
     startOffset = newStartOffset;
     endOffset = newEndOffset;
     type = newType;
@@ -504,14 +504,14 @@ public class Token extends TermAttribute
   }
 
   /** Shorthand for calling {@link #clear},
-   *  {@link #setTermBuffer(String)},
+   *  {@link #append(CharSequence)},
    *  {@link #setStartOffset},
    *  {@link #setEndOffset}
    *  {@link #setType} on Token.DEFAULT_TYPE
    *  @return this Token instance */
   public Token reinit(String newTerm, int newStartOffset, int newEndOffset) {
-    clearNoTermBuffer();
-    setTermBuffer(newTerm);
+    clear();
+    append(newTerm);
     startOffset = newStartOffset;
     endOffset = newEndOffset;
     type = DEFAULT_TYPE;
@@ -519,14 +519,14 @@ public class Token extends TermAttribute
   }
 
   /** Shorthand for calling {@link #clear},
-   *  {@link #setTermBuffer(String, int, int)},
+   *  {@link #append(CharSequence, int, int)},
    *  {@link #setStartOffset},
    *  {@link #setEndOffset}
    *  {@link #setType} on Token.DEFAULT_TYPE
    *  @return this Token instance */
   public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
-    clearNoTermBuffer();
-    setTermBuffer(newTerm, newTermOffset, newTermLength);
+    clear();
+    append(newTerm, newTermOffset, newTermOffset + newTermLength);
     startOffset = newStartOffset;
     endOffset = newEndOffset;
     type = DEFAULT_TYPE;
@@ -538,7 +538,7 @@ public class Token extends TermAttribute
    * @param prototype
    */
   public void reinit(Token prototype) {
-    setTermBuffer(prototype.termBuffer(), 0, prototype.termLength());
+    copyBuffer(prototype.buffer(), 0, prototype.length());
     positionIncrement = prototype.positionIncrement;
     flags = prototype.flags;
     startOffset = prototype.startOffset;
@@ -553,7 +553,7 @@ public class Token extends TermAttribute
    * @param newTerm
    */
   public void reinit(Token prototype, String newTerm) {
-    setTermBuffer(newTerm);
+    setEmpty().append(newTerm);
     positionIncrement = prototype.positionIncrement;
     flags = prototype.flags;
     startOffset = prototype.startOffset;
@@ -570,7 +570,7 @@ public class Token extends TermAttribute
    * @param length
    */
   public void reinit(Token prototype, char[] newTermBuffer, int offset, int length) {
-    setTermBuffer(newTermBuffer, offset, length);
+    copyBuffer(newTermBuffer, offset, length);
     positionIncrement = prototype.positionIncrement;
     flags = prototype.flags;
     startOffset = prototype.startOffset;

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,2 +1,2 @@
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:909334
-/lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java?rev=924791&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java Thu Mar 18 14:13:01 2010
@@ -0,0 +1,71 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+/**
+ * The term text of a Token.
+ */
+public interface CharTermAttribute extends Attribute, CharSequence, Appendable {
+  
+  /** Copies the contents of buffer, starting at offset for
+   *  length characters, into the termBuffer array.
+   *  @param buffer the buffer to copy
+   *  @param offset the index in the buffer of the first character to copy
+   *  @param length the number of characters to copy
+   */
+  public void copyBuffer(char[] buffer, int offset, int length);
+  
+  /** Returns the internal termBuffer character array which
+   *  you can then directly alter.  If the array is too
+   *  small for your token, use {@link
+   *  #resizeBuffer(int)} to increase it.  After
+   *  altering the buffer be sure to call {@link
+   *  #setLength} to record the number of valid
+   *  characters that were placed into the termBuffer. */
+  public char[] buffer();
+
+  /** Grows the termBuffer to at least size newSize, preserving the
+   *  existing content.
+   *  @param newSize minimum size of the new termBuffer
+   *  @return newly created termBuffer with length >= newSize
+   */
+  public char[] resizeBuffer(int newSize);
+
+  /** Set number of valid characters (length of the term) in
+   *  the termBuffer array. Use this to truncate the termBuffer
+   *  or to synchronize with external manipulation of the termBuffer.
+   *  Note: to grow the size of the array,
+   *  use {@link #resizeBuffer(int)} first.
+   *  @param length the truncated length
+   */
+  public CharTermAttribute setLength(int length);
+  
+  /** Sets the length of the termBuffer to zero.
+   * Use this method before appending contents
+   * using the {@link Appendable} interface.
+   */
+  public CharTermAttribute setEmpty();
+  
+  // the following methods are redefined to get rid of IOException declaration:
+  public CharTermAttribute append(CharSequence csq);
+  public CharTermAttribute append(CharSequence csq, int start, int end);
+  public CharTermAttribute append(char c);
+
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java?rev=924791&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java Thu Mar 18 14:13:01 2010
@@ -0,0 +1,256 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Serializable;
+import java.nio.CharBuffer;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.UnicodeUtil;
+
+/**
+ * The term text of a Token.
+ */
+public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttribute, TermAttribute, TermToBytesRefAttribute, Cloneable, Serializable {
+  private static int MIN_BUFFER_SIZE = 10;
+  
+  private char[] termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
+  private int termLength = 0;
+  
+  @Deprecated
+  public String term() {
+    // don't delegate to toString() here!
+    return new String(termBuffer, 0, termLength);
+  }
+
+  public void copyBuffer(char[] buffer, int offset, int length) {
+    growTermBuffer(length);
+    System.arraycopy(buffer, offset, termBuffer, 0, length);
+    termLength = length;
+  }
+
+  @Deprecated
+  public void setTermBuffer(char[] buffer, int offset, int length) {
+    copyBuffer(buffer, offset, length);
+  }
+
+  @Deprecated
+  public void setTermBuffer(String buffer) {
+    int length = buffer.length();
+    growTermBuffer(length);
+    buffer.getChars(0, length, termBuffer, 0);
+    termLength = length;
+  }
+
+  @Deprecated
+  public void setTermBuffer(String buffer, int offset, int length) {
+    assert offset <= buffer.length();
+    assert offset + length <= buffer.length();
+    growTermBuffer(length);
+    buffer.getChars(offset, offset + length, termBuffer, 0);
+    termLength = length;
+  }
+
+  public char[] buffer() {
+    return termBuffer;
+  }
+
+  @Deprecated
+  public char[] termBuffer() {
+    return termBuffer;
+  }
+  
+  public char[] resizeBuffer(int newSize) {
+    if (termBuffer == null) {
+      // The buffer is always at least MIN_BUFFER_SIZE
+      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)]; 
+    } else {
+      if(termBuffer.length < newSize){
+        // Not big enough; create a new array with slight
+        // over allocation and preserve content
+        final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
+        System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
+        termBuffer = newCharBuffer;
+      }
+    } 
+    return termBuffer;   
+  }
+
+  @Deprecated
+  public char[] resizeTermBuffer(int newSize) {
+    return resizeBuffer(newSize);
+  }
+  
+  private void growTermBuffer(int newSize) {
+    if (termBuffer == null) {
+      // The buffer is always at least MIN_BUFFER_SIZE
+      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)];   
+    } else {
+      if(termBuffer.length < newSize){
+        // Not big enough; create a new array with slight
+        // over allocation:
+        termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
+      }
+    } 
+  }
+  
+  @Deprecated
+  public int termLength() {
+    return termLength;
+  }
+
+  public CharTermAttribute setLength(int length) {
+    if (length > termBuffer.length)
+      throw new IllegalArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.length + ")");
+    termLength = length;
+    return this;
+  }
+  
+  public CharTermAttribute setEmpty() {
+    termLength = 0;
+    return this;
+  }
+  
+  @Deprecated
+  public void setTermLength(int length) {
+    setLength(length);
+  }
+  
+  // *** TermToBytesRefAttribute interface ***
+  public int toBytesRef(BytesRef target) {
+    // nocommit: Maybe assume that bytes is already initialized? TermsHashPerField ensures this.
+    if (target.bytes == null) {
+      target.bytes = new byte[termLength * 4];
+    }
+    return UnicodeUtil.UTF16toUTF8WithHash(termBuffer, 0, termLength, target);
+  }
+  
+  // *** CharSequence interface ***
+  public int length() {
+    return termLength;
+  }
+  
+  public char charAt(int index) {
+    if (index >= termLength)
+      throw new IndexOutOfBoundsException();
+    return termBuffer[index];
+  }
+  
+  public CharSequence subSequence(final int start, final int end) {
+    if (start > termLength || end > termLength)
+      throw new IndexOutOfBoundsException();
+    return new String(termBuffer, start, end - start);
+  }
+  
+  // *** Appendable interface ***
+  public CharTermAttribute append(CharSequence csq) {
+    return append(csq, 0, csq.length());
+  }
+  
+  public CharTermAttribute append(CharSequence csq, int start, int end) {
+    resizeBuffer(termLength + end - start);
+    if (csq instanceof String) {
+      ((String) csq).getChars(start, end, termBuffer, termLength);
+    } else if (csq instanceof StringBuilder) {
+      ((StringBuilder) csq).getChars(start, end, termBuffer, termLength);
+    } else if (csq instanceof StringBuffer) {
+      ((StringBuffer) csq).getChars(start, end, termBuffer, termLength);
+    } else if (csq instanceof CharBuffer && ((CharBuffer) csq).hasArray()) {
+      final CharBuffer cb = (CharBuffer) csq;
+      System.arraycopy(cb.array(), cb.arrayOffset() + cb.position() + start, termBuffer, termLength, end - start);
+    } else {
+      while (start < end)
+        termBuffer[termLength++] = csq.charAt(start++);
+      // no fall-through here, as termLength is updated!
+      return this;
+    }
+    termLength += end - start;
+    return this;
+  }
+  
+  public CharTermAttribute append(char c) {
+    resizeBuffer(termLength + 1)[termLength++] = c;
+    return this;
+  }
+  
+  // *** AttributeImpl ***
+
+  @Override
+  public int hashCode() {
+    int code = termLength;
+    code = code * 31 + ArrayUtil.hashCode(termBuffer, 0, termLength);
+    return code;
+  }
+
+  @Override
+  public void clear() {
+    termLength = 0;    
+  }
+
+  @Override
+  public Object clone() {
+    CharTermAttributeImpl t = (CharTermAttributeImpl)super.clone();
+    // Do a deep clone
+    if (termBuffer != null) {
+      t.termBuffer = termBuffer.clone();
+    }
+    return t;
+  }
+  
+  @Override
+  public boolean equals(Object other) {
+    if (other == this) {
+      return true;
+    }
+    
+    if (other instanceof CharTermAttributeImpl) {
+      final CharTermAttributeImpl o = ((CharTermAttributeImpl) other);
+      if (termLength != o.termLength)
+        return false;
+      for(int i=0;i<termLength;i++) {
+        if (termBuffer[i] != o.termBuffer[i]) {
+          return false;
+        }
+      }
+      return true;
+    }
+    
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    // nocommit: CharSequence requires that only the contents are returned, but this is orginal code: "term=" + new String(termBuffer, 0, termLength)
+    return new String(termBuffer, 0, termLength);
+  }
+  
+  @Override
+  public void copyTo(AttributeImpl target) {
+    if (target instanceof CharTermAttribute) {
+      CharTermAttribute t = (CharTermAttribute) target;
+      t.copyBuffer(termBuffer, 0, termLength);
+    } else {
+      TermAttribute t = (TermAttribute) target;
+      t.setTermBuffer(termBuffer, 0, termLength);
+    }
+  }
+
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttribute.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttribute.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttribute.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttribute.java Thu Mar 18 14:13:01 2010
@@ -21,7 +21,9 @@ import org.apache.lucene.util.Attribute;
 
 /**
  * The term text of a Token.
+ * @deprecated Use {@link CharTermAttribute} instead.
  */
+@Deprecated
 public interface TermAttribute extends Attribute {
   /** Returns the Token's term text.
    * 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java Thu Mar 18 14:13:01 2010
@@ -17,211 +17,11 @@ package org.apache.lucene.analysis.token
  * limitations under the License.
  */
 
-import java.io.Serializable;
-
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.AttributeImpl;
-import org.apache.lucene.util.RamUsageEstimator;
-
 /**
  * The term text of a Token.
+ * @deprecated This class is only available for AttributeSource
+ * to be able to load an old TermAttribute implementation class.
  */
-public class TermAttributeImpl extends AttributeImpl implements TermAttribute, Cloneable, Serializable {
-  private static int MIN_BUFFER_SIZE = 10;
-  
-  private char[] termBuffer;
-  private int termLength;
-  
-  /** Returns the Token's term text.
-   * 
-   * This method has a performance penalty
-   * because the text is stored internally in a char[].  If
-   * possible, use {@link #termBuffer()} and {@link
-   * #termLength()} directly instead.  If you really need a
-   * String, use this method, which is nothing more than
-   * a convenience call to <b>new String(token.termBuffer(), 0, token.termLength())</b>
-   */
-  public String term() {
-    initTermBuffer();
-    return new String(termBuffer, 0, termLength);
-  }
-
-  /** Copies the contents of buffer, starting at offset for
-   *  length characters, into the termBuffer array.
-   *  @param buffer the buffer to copy
-   *  @param offset the index in the buffer of the first character to copy
-   *  @param length the number of characters to copy
-   */
-  public void setTermBuffer(char[] buffer, int offset, int length) {
-    growTermBuffer(length);
-    System.arraycopy(buffer, offset, termBuffer, 0, length);
-    termLength = length;
-  }
-
-  /** Copies the contents of buffer into the termBuffer array.
-   *  @param buffer the buffer to copy
-   */
-  public void setTermBuffer(String buffer) {
-    int length = buffer.length();
-    growTermBuffer(length);
-    buffer.getChars(0, length, termBuffer, 0);
-    termLength = length;
-  }
-
-  /** Copies the contents of buffer, starting at offset and continuing
-   *  for length characters, into the termBuffer array.
-   *  @param buffer the buffer to copy
-   *  @param offset the index in the buffer of the first character to copy
-   *  @param length the number of characters to copy
-   */
-  public void setTermBuffer(String buffer, int offset, int length) {
-    assert offset <= buffer.length();
-    assert offset + length <= buffer.length();
-    growTermBuffer(length);
-    buffer.getChars(offset, offset + length, termBuffer, 0);
-    termLength = length;
-  }
-
-  /** Returns the internal termBuffer character array which
-   *  you can then directly alter.  If the array is too
-   *  small for your token, use {@link
-   *  #resizeTermBuffer(int)} to increase it.  After
-   *  altering the buffer be sure to call {@link
-   *  #setTermLength} to record the number of valid
-   *  characters that were placed into the termBuffer. */
-  public char[] termBuffer() {
-    initTermBuffer();
-    return termBuffer;
-  }
-
-  /** Grows the termBuffer to at least size newSize, preserving the
-   *  existing content. Note: If the next operation is to change
-   *  the contents of the term buffer use
-   *  {@link #setTermBuffer(char[], int, int)},
-   *  {@link #setTermBuffer(String)}, or
-   *  {@link #setTermBuffer(String, int, int)}
-   *  to optimally combine the resize with the setting of the termBuffer.
-   *  @param newSize minimum size of the new termBuffer
-   *  @return newly created termBuffer with length >= newSize
-   */
-  public char[] resizeTermBuffer(int newSize) {
-    if (termBuffer == null) {
-      // The buffer is always at least MIN_BUFFER_SIZE
-      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)]; 
-    } else {
-      if(termBuffer.length < newSize){
-        // Not big enough; create a new array with slight
-        // over allocation and preserve content
-        final char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
-        System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
-        termBuffer = newCharBuffer;
-      }
-    } 
-    return termBuffer;   
-  }
-
-
-  /** Allocates a buffer char[] of at least newSize, without preserving the existing content.
-   * its always used in places that set the content 
-   *  @param newSize minimum size of the buffer
-   */
-  private void growTermBuffer(int newSize) {
-    if (termBuffer == null) {
-      // The buffer is always at least MIN_BUFFER_SIZE
-      termBuffer = new char[ArrayUtil.oversize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize, RamUsageEstimator.NUM_BYTES_CHAR)];   
-    } else {
-      if(termBuffer.length < newSize){
-        // Not big enough; create a new array with slight
-        // over allocation:
-        termBuffer = new char[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_CHAR)];
-      }
-    } 
-  }
-  
-  private void initTermBuffer() {
-    if (termBuffer == null) {
-      termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, RamUsageEstimator.NUM_BYTES_CHAR)];
-      termLength = 0;
-    }
-  }
-
-  /** Return number of valid characters (length of the term)
-   *  in the termBuffer array. */
-  public int termLength() {
-    return termLength;
-  }
-
-  /** Set number of valid characters (length of the term) in
-   *  the termBuffer array. Use this to truncate the termBuffer
-   *  or to synchronize with external manipulation of the termBuffer.
-   *  Note: to grow the size of the array,
-   *  use {@link #resizeTermBuffer(int)} first.
-   *  @param length the truncated length
-   */
-  public void setTermLength(int length) {
-    initTermBuffer();
-    if (length > termBuffer.length)
-      throw new IllegalArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.length + ")");
-    termLength = length;
-  }
-
-  @Override
-  public int hashCode() {
-    initTermBuffer();
-    int code = termLength;
-    code = code * 31 + ArrayUtil.hashCode(termBuffer, 0, termLength);
-    return code;
-  }
-
-  @Override
-  public void clear() {
-    termLength = 0;    
-  }
-
-  @Override
-  public Object clone() {
-    TermAttributeImpl t = (TermAttributeImpl)super.clone();
-    // Do a deep clone
-    if (termBuffer != null) {
-      t.termBuffer = termBuffer.clone();
-    }
-    return t;
-  }
-  
-  @Override
-  public boolean equals(Object other) {
-    if (other == this) {
-      return true;
-    }
-    
-    if (other instanceof TermAttributeImpl) {
-      initTermBuffer();
-      TermAttributeImpl o = ((TermAttributeImpl) other);
-      o.initTermBuffer();
-      
-      if (termLength != o.termLength)
-        return false;
-      for(int i=0;i<termLength;i++) {
-        if (termBuffer[i] != o.termBuffer[i]) {
-          return false;
-        }
-      }
-      return true;
-    }
-    
-    return false;
-  }
-
-  @Override
-  public String toString() {
-    initTermBuffer();
-    return "term=" + new String(termBuffer, 0, termLength);
-  }
-  
-  @Override
-  public void copyTo(AttributeImpl target) {
-    initTermBuffer();
-    TermAttribute t = (TermAttribute) target;
-    t.setTermBuffer(termBuffer, 0, termLength);
-  }
+@Deprecated
+public class TermAttributeImpl extends CharTermAttributeImpl {
 }

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java?rev=924791&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java Thu Mar 18 14:13:01 2010
@@ -0,0 +1,32 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * This attribute is requested by TermsHashPerField to index the contents.
+ * This attribute has no real state, it should be implemented in addition to
+ * {@link CharTermAttribute}, to support indexing the term text as
+ * UTF-8 bytes.
+ */
+public interface TermToBytesRefAttribute extends Attribute {
+  /** Copies the token's term text into the given {@link BytesRef}. It returns the hashCode for TermsHash */
+  public int toBytesRef(BytesRef bytes);
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocInverterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocInverterPerThread.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocInverterPerThread.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocInverterPerThread.java Thu Mar 18 14:13:01 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /** This is a DocFieldConsumer that inverts each field,
  *  separately, from a Document, and accepts a
@@ -34,16 +34,16 @@ final class DocInverterPerThread extends
   final SingleTokenAttributeSource singleToken = new SingleTokenAttributeSource();
   
   static class SingleTokenAttributeSource extends AttributeSource {
-    final TermAttribute termAttribute;
+    final CharTermAttribute termAttribute;
     final OffsetAttribute offsetAttribute;
     
     private SingleTokenAttributeSource() {
-      termAttribute = addAttribute(TermAttribute.class);
+      termAttribute = addAttribute(CharTermAttribute.class);
       offsetAttribute = addAttribute(OffsetAttribute.class);
     }
     
     public void reinit(String stringValue, int startOffset,  int endOffset) {
-      termAttribute.setTermBuffer(stringValue);
+      termAttribute.setEmpty().append(stringValue);
       offsetAttribute.setOffset(startOffset, endOffset);
     }
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java Thu Mar 18 14:13:01 2010
@@ -22,6 +22,7 @@ import java.util.Arrays;
 import java.util.Comparator;
 
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.BytesRef;
@@ -33,7 +34,7 @@ final class TermsHashPerField extends In
   final TermsHashPerThread perThread;
   final DocumentsWriter.DocState docState;
   final FieldInvertState fieldState;
-  TermAttribute termAtt;
+  TermToBytesRefAttribute termAtt;
 
   // Copied from our perThread
   final IntBlockPool intPool;
@@ -259,7 +260,14 @@ final class TermsHashPerField extends In
 
   @Override
   void start(Fieldable f) {
-    termAtt = fieldState.attributeSource.addAttribute(TermAttribute.class);
+    if (fieldState.attributeSource.hasAttribute(TermToBytesRefAttribute.class)) {
+      termAtt = fieldState.attributeSource.getAttribute(TermToBytesRefAttribute.class);
+    } else if (fieldState.attributeSource.hasAttribute(TermAttribute.class)) {
+      perThread.legacyTermAttributeWrapper.setTermAttribute(fieldState.attributeSource.getAttribute(TermAttribute.class));
+      termAtt = perThread.legacyTermAttributeWrapper;
+    } else {
+      throw new IllegalArgumentException("Could not find a term attribute (that implements TermToBytesRefAttribute) in the TokenStream");
+    }
     consumer.start(f);
     if (nextPerField != null) {
       nextPerField.start(f);
@@ -358,13 +366,8 @@ final class TermsHashPerField extends In
     // We are first in the chain so we must "intern" the
     // term text into textStart address
 
-    // Get the text of this term.
-    final char[] tokenText = termAtt.termBuffer();
-    final int tokenTextLen = termAtt.termLength();
-    
-    //System.out.println("\nfield=" + fieldInfo.name + " add text=" + new String(tokenText, 0, tokenTextLen) + " len=" + tokenTextLen);
-    
-    int code = UnicodeUtil.UTF16toUTF8WithHash(tokenText, 0, tokenTextLen, utf8);
+    // Get the text & hash of this term.
+    int code = termAtt.toBytesRef(utf8);
 
     int hashPos = code & postingsHashMask;
 
@@ -399,7 +402,13 @@ final class TermsHashPerField extends In
           // other behavior is wanted (pruning the term
           // to a prefix, throwing an exception, etc).
           if (docState.maxTermPrefix == null) {
-            docState.maxTermPrefix = new String(tokenText, 0, 30);
+            final int saved = utf8.length;
+            try {
+              utf8.length = Math.min(30, DocumentsWriter.MAX_TERM_LENGTH_UTF8);
+              docState.maxTermPrefix = utf8.toString();
+            } finally {
+              utf8.length = saved;
+            }
           }
 
           consumer.skippingLongTerm();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java Thu Mar 18 14:13:01 2010
@@ -18,6 +18,9 @@ package org.apache.lucene.index;
  */
 
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 
 import java.io.IOException;
 
@@ -42,6 +45,23 @@ final class TermsHashPerThread extends I
 
   // Used by perField:
   final BytesRef utf8 = new BytesRef(10);
+  
+  final LegacyTermAttributeWrapper legacyTermAttributeWrapper = new LegacyTermAttributeWrapper();
+  
+  /** This class is used to wrap a legacy TermAttribute without support for {@link TermToBytesRefAttribute}. */
+  @Deprecated
+  static class LegacyTermAttributeWrapper implements TermToBytesRefAttribute {
+    private TermAttribute termAtt = null;
+  
+    void setTermAttribute(TermAttribute termAtt) {
+      this.termAtt = termAtt;
+    }
+  
+    public int toBytesRef(BytesRef target) {
+      assert target.bytes != null : "target byteref must be != null, because utf8 is used here";
+      return UnicodeUtil.UTF16toUTF8WithHash(termAtt.termBuffer(), 0, termAtt.termLength(), target);
+    }
+  }
 
   public TermsHashPerThread(DocInverterPerThread docInverterPerThread, final TermsHash termsHash, final TermsHash nextTermsHash, final TermsHashPerThread primaryPerThread) {
     docState = docInverterPerThread.docState;

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,2 +1,2 @@
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334
-/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,4 +1,4 @@
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:829134,829881,831036,896850,909334
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:880793,896906
-/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Added: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java?rev=924791&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java (added)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java Thu Mar 18 14:13:01 2010
@@ -0,0 +1,177 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.LuceneTestCase;
+import java.nio.CharBuffer;
+import java.util.Formatter;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+public class TestCharTermAttributeImpl extends LuceneTestCase {
+
+  public TestCharTermAttributeImpl(String name) {
+    super(name);
+  }
+
+  public void testResize() {
+    CharTermAttributeImpl t = new CharTermAttributeImpl();
+    char[] content = "hello".toCharArray();
+    t.copyBuffer(content, 0, content.length);
+    for (int i = 0; i < 2000; i++)
+    {
+      t.resizeBuffer(i);
+      assertTrue(i <= t.buffer().length);
+      assertEquals("hello", t.toString());
+    }
+  }
+
+  public void testGrow() {
+    CharTermAttributeImpl t = new CharTermAttributeImpl();
+    StringBuilder buf = new StringBuilder("ab");
+    for (int i = 0; i < 20; i++)
+    {
+      char[] content = buf.toString().toCharArray();
+      t.copyBuffer(content, 0, content.length);
+      assertEquals(buf.length(), t.length());
+      assertEquals(buf.toString(), t.toString());
+      buf.append(buf.toString());
+    }
+    assertEquals(1048576, t.length());
+
+    // now as a StringBuilder, first variant
+    t = new CharTermAttributeImpl();
+    buf = new StringBuilder("ab");
+    for (int i = 0; i < 20; i++)
+    {
+      t.setEmpty().append(buf);
+      assertEquals(buf.length(), t.length());
+      assertEquals(buf.toString(), t.toString());
+      buf.append(t);
+    }
+    assertEquals(1048576, t.length());
+
+    // Test for slow growth to a long term
+    t = new CharTermAttributeImpl();
+    buf = new StringBuilder("a");
+    for (int i = 0; i < 20000; i++)
+    {
+      t.setEmpty().append(buf);
+      assertEquals(buf.length(), t.length());
+      assertEquals(buf.toString(), t.toString());
+      buf.append("a");
+    }
+    assertEquals(20000, t.length());
+  }
+
+  public void testToString() throws Exception {
+    char[] b = {'a', 'l', 'o', 'h', 'a'};
+    CharTermAttributeImpl t = new CharTermAttributeImpl();
+    t.copyBuffer(b, 0, 5);
+    assertEquals("aloha", t.toString());
+
+    t.setEmpty().append("hi there");
+    assertEquals("hi there", t.toString());
+  }
+
+  public void testClone() throws Exception {
+    CharTermAttributeImpl t = new CharTermAttributeImpl();
+    char[] content = "hello".toCharArray();
+    t.copyBuffer(content, 0, 5);
+    char[] buf = t.buffer();
+    CharTermAttributeImpl copy = (CharTermAttributeImpl) TestSimpleAttributeImpls.assertCloneIsEqual(t);
+    assertEquals(t.toString(), copy.toString());
+    assertNotSame(buf, copy.buffer());
+  }
+  
+  public void testEquals() throws Exception {
+    CharTermAttributeImpl t1a = new CharTermAttributeImpl();
+    char[] content1a = "hello".toCharArray();
+    t1a.copyBuffer(content1a, 0, 5);
+    CharTermAttributeImpl t1b = new CharTermAttributeImpl();
+    char[] content1b = "hello".toCharArray();
+    t1b.copyBuffer(content1b, 0, 5);
+    CharTermAttributeImpl t2 = new CharTermAttributeImpl();
+    char[] content2 = "hello2".toCharArray();
+    t2.copyBuffer(content2, 0, 6);
+    assertTrue(t1a.equals(t1b));
+    assertFalse(t1a.equals(t2));
+    assertFalse(t2.equals(t1b));
+  }
+  
+  public void testCopyTo() throws Exception {
+    CharTermAttributeImpl t = new CharTermAttributeImpl();
+    CharTermAttributeImpl copy = (CharTermAttributeImpl) TestSimpleAttributeImpls.assertCopyIsEqual(t);
+    assertEquals("", t.toString());
+    assertEquals("", copy.toString());
+
+    t = new CharTermAttributeImpl();
+    char[] content = "hello".toCharArray();
+    t.copyBuffer(content, 0, 5);
+    char[] buf = t.buffer();
+    copy = (CharTermAttributeImpl) TestSimpleAttributeImpls.assertCopyIsEqual(t);
+    assertEquals(t.toString(), copy.toString());
+    assertNotSame(buf, copy.buffer());
+  }
+  
+  public void testCharSequenceInterface() {
+    final String s = "0123456789"; 
+    final CharTermAttributeImpl t = new CharTermAttributeImpl();
+    t.append(s);
+    
+    assertEquals(s.length(), t.length());
+    assertEquals("12", t.subSequence(1,3).toString());
+    assertEquals(s, t.subSequence(0,s.length()).toString());
+    
+    assertTrue(Pattern.matches("01\\d+", t));
+    assertTrue(Pattern.matches("34", t.subSequence(3,5)));
+    
+    assertEquals(s.subSequence(3,7).toString(), t.subSequence(3,7).toString());
+    
+    for (int i = 0; i < s.length(); i++) {
+      assertTrue(t.charAt(i) == s.charAt(i));
+    }
+  }
+
+  public void testAppendableInterface() {
+    CharTermAttributeImpl t = new CharTermAttributeImpl();
+    Formatter formatter = new Formatter(t, Locale.US);
+    formatter.format("%d", 1234);
+    assertEquals("1234", t.toString());
+    formatter.format("%d", 5678);
+    assertEquals("12345678", t.toString());
+    t.append('9');
+    assertEquals("123456789", t.toString());
+    t.append("0");
+    assertEquals("1234567890", t.toString());
+    t.append("0123456789", 1, 3);
+    assertEquals("123456789012", t.toString());
+    t.append(CharBuffer.wrap("0123456789".toCharArray()), 3, 5);
+    assertEquals("12345678901234", t.toString());
+    t.append(t);
+    assertEquals("1234567890123412345678901234", t.toString());
+    t.append(new StringBuilder("0123456789"), 5, 7);
+    assertEquals("123456789012341234567890123456", t.toString());
+    t.append(new StringBuffer(t));
+    assertEquals("123456789012341234567890123456123456789012341234567890123456", t.toString());
+    // very wierd, to test if a subSlice is wrapped correct :)
+    t.setEmpty().append(CharBuffer.wrap("0123456789".toCharArray(), 3, 5) /* "34" */, 1, 2);
+    assertEquals("4", t.toString());
+  }
+  
+}

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java Thu Mar 18 14:13:01 2010
@@ -22,6 +22,7 @@ import org.apache.lucene.util.AttributeI
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.AttributeSource.AttributeFactory;
 
+@Deprecated
 public class TestSimpleAttributeImpls extends LuceneTestCase {
 
   public TestSimpleAttributeImpls(String name) {

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java?rev=924791&r1=924790&r2=924791&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/tokenattributes/TestTermAttributeImpl.java Thu Mar 18 14:13:01 2010
@@ -107,10 +107,10 @@ public class TestTermAttributeImpl exten
     char[] b = {'a', 'l', 'o', 'h', 'a'};
     TermAttributeImpl t = new TermAttributeImpl();
     t.setTermBuffer(b, 0, 5);
-    assertEquals("term=aloha", t.toString());
+    assertEquals("aloha", t.toString());
 
     t.setTermBuffer("hi there");
-    assertEquals("term=hi there", t.toString());
+    assertEquals("hi there", t.toString());
   }
 
   public void testMixedStringArray() throws Exception {

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,4 +1,4 @@
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:829134,829881,831036,896850,909334
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880793,896906
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,4 +1,4 @@
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880793,896906
-/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 18 14:13:01 2010
@@ -1,4 +1,4 @@
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880793,896906
-/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731
+/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-825292,826213-833960,880727-886190,889185,889614-916543,916596,916685,916755,916769,917019,917148,917154,917161,917203-917204,919060,919119,919718,919731,919869,920237-920499,922013,922277,922525,922528,922583,922593,922602,922624,922797,922799,922806,922886-922887,923112,923238,924151,924207,924286,924321-924731,924781



Mime
View raw message