lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1181188 [6/8] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/modules/grouping/ dev-tools/idea/solr/ dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea...
Date Mon, 10 Oct 2011 19:58:32 GMT
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Mon Oct 10 19:58:24 2011
@@ -47,8 +47,6 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.FloatsRef;
-import org.apache.lucene.util.LongsRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 import org.junit.Before;
@@ -136,7 +134,6 @@ public class TestDocValuesIndexing exten
     Collections.shuffle(values, random);
     ValueType first = values.get(0);
     ValueType second = values.get(1);
-    String msg = "[first=" + first.name() + ", second=" + second.name() + "]";
     // index first index
     Directory d_1 = newDirectory();
     IndexWriter w_1 = new IndexWriter(d_1, writerConfig(random.nextBoolean()));
@@ -171,36 +168,66 @@ public class TestDocValuesIndexing exten
     // check values
     
     IndexReader merged = IndexReader.open(w, true);
-    ValuesEnum vE_1 = getValuesEnum(getDocValues(r_1, first.name()));
-    ValuesEnum vE_2 = getValuesEnum(getDocValues(r_2, second.name()));
-    ValuesEnum vE_1_merged = getValuesEnum(getDocValues(merged, first.name()));
-    ValuesEnum vE_2_merged = getValuesEnum(getDocValues(merged, second
+    Source source_1 = getSource(getDocValues(r_1, first.name()));
+    Source source_2 = getSource(getDocValues(r_2, second.name()));
+    Source source_1_merged = getSource(getDocValues(merged, first.name()));
+    Source source_2_merged = getSource(getDocValues(merged, second
         .name()));
-    switch (second) { // these variants don't advance over missing values
-    case BYTES_FIXED_STRAIGHT:
-    case BYTES_VAR_STRAIGHT:
-    case FLOAT_32:
-    case FLOAT_64:
-    case VAR_INTS:
-    case FIXED_INTS_16:
-    case FIXED_INTS_32:
-    case FIXED_INTS_64:
-    case FIXED_INTS_8:
-      assertEquals(msg, valuesPerIndex-1, vE_2_merged.advance(valuesPerIndex-1));
+    for (int i = 0; i < r_1.maxDoc(); i++) {
+      switch (first) {
+      case BYTES_FIXED_DEREF:
+      case BYTES_FIXED_STRAIGHT:
+      case BYTES_VAR_DEREF:
+      case BYTES_VAR_STRAIGHT:
+      case BYTES_FIXED_SORTED:
+      case BYTES_VAR_SORTED:
+        assertEquals(source_1.getBytes(i, new BytesRef()),
+            source_1_merged.getBytes(i, new BytesRef()));
+        break;
+      case FIXED_INTS_16:
+      case FIXED_INTS_32:
+      case FIXED_INTS_64:
+      case FIXED_INTS_8:
+      case VAR_INTS:
+        assertEquals(source_1.getInt(i), source_1_merged.getInt(i));
+        break;
+      case FLOAT_32:
+      case FLOAT_64:
+        assertEquals(source_1.getFloat(i), source_1_merged.getFloat(i), 0.0d);
+        break;
+      default:
+        fail("unkonwn " + first);
+      }
     }
-    
-    for (int i = 0; i < valuesPerIndex; i++) {
-      assertEquals(msg, i, vE_1.nextDoc());
-      assertEquals(msg, i, vE_1_merged.nextDoc());
-
-      assertEquals(msg, i, vE_2.nextDoc());
-      assertEquals(msg, i + valuesPerIndex, vE_2_merged.nextDoc());
-    }
-    assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_1.nextDoc());
-    assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_2.nextDoc());
-    assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_1_merged.advance(valuesPerIndex*2));
-    assertEquals(msg, ValuesEnum.NO_MORE_DOCS, vE_2_merged.nextDoc());
 
+    for (int i = r_1.maxDoc(); i < merged.maxDoc(); i++) {
+      switch (second) {
+      case BYTES_FIXED_DEREF:
+      case BYTES_FIXED_STRAIGHT:
+      case BYTES_VAR_DEREF:
+      case BYTES_VAR_STRAIGHT:
+      case BYTES_FIXED_SORTED:
+      case BYTES_VAR_SORTED:
+        assertEquals(source_2.getBytes(i - r_1.maxDoc(), new BytesRef()),
+            source_2_merged.getBytes(i, new BytesRef()));
+        break;
+      case FIXED_INTS_16:
+      case FIXED_INTS_32:
+      case FIXED_INTS_64:
+      case FIXED_INTS_8:
+      case VAR_INTS:
+        assertEquals(source_2.getInt(i - r_1.maxDoc()),
+            source_2_merged.getInt(i));
+        break;
+      case FLOAT_32:
+      case FLOAT_64:
+        assertEquals(source_2.getFloat(i - r_1.maxDoc()),
+            source_2_merged.getFloat(i), 0.0d);
+        break;
+      default:
+        fail("unkonwn " + first);
+      }
+    }
     // close resources
     r_1.close();
     r_2.close();
@@ -260,22 +287,12 @@ public class TestDocValuesIndexing exten
           assertEquals("index " + i, 0, value);
         }
 
-        ValuesEnum intsEnum = getValuesEnum(intsReader);
-        assertTrue(intsEnum.advance(base) >= base);
-
-        intsEnum = getValuesEnum(intsReader);
-        LongsRef enumRef = intsEnum.getInt();
-
         int expected = 0;
         for (int i = base; i < r.numDocs(); i++, expected++) {
           while (deleted.get(expected)) {
             expected++;
           }
-          assertEquals("advance failed at index: " + i + " of " + r.numDocs()
-              + " docs", i, intsEnum.advance(i));
           assertEquals(val + " mod: " + mod + " index: " +  i, expected%mod, ints.getInt(i));
-          assertEquals(expected%mod, enumRef.get());
-
         }
       }
         break;
@@ -289,20 +306,11 @@ public class TestDocValuesIndexing exten
           assertEquals(val + " failed for doc: " + i + " base: " + base,
               0.0d, value, 0.0d);
         }
-        ValuesEnum floatEnum = getValuesEnum(floatReader);
-        assertTrue(floatEnum.advance(base) >= base);
-
-        floatEnum = getValuesEnum(floatReader);
-        FloatsRef enumRef = floatEnum.getFloat();
         int expected = 0;
         for (int i = base; i < r.numDocs(); i++, expected++) {
           while (deleted.get(expected)) {
             expected++;
           }
-          assertEquals("advance failed at index: " + i + " of " + r.numDocs()
-              + " docs base:" + base, i, floatEnum.advance(i));
-          assertEquals(floatEnum.getClass() + " index " + i, 2.0 * expected,
-              enumRef.get(), 0.00001);
           assertEquals("index " + i, 2.0 * expected, floats.getFloat(i),
               0.00001);
         }
@@ -320,7 +328,7 @@ public class TestDocValuesIndexing exten
     w.close();
     d.close();
   }
-
+  
   public void runTestIndexBytes(IndexWriterConfig cfg, boolean withDeletions)
       throws CorruptIndexException, LockObtainFailedException, IOException {
     final Directory d = newDirectory();
@@ -353,6 +361,8 @@ public class TestDocValuesIndexing exten
         switch (byteIndexValue) {
         case BYTES_VAR_STRAIGHT:
         case BYTES_FIXED_STRAIGHT:
+        case BYTES_FIXED_DEREF:
+        case BYTES_FIXED_SORTED:
           // fixed straight returns bytesref with zero bytes all of fixed
           // length
           assertNotNull("expected none null - " + msg, br);
@@ -365,23 +375,13 @@ public class TestDocValuesIndexing exten
             }
           }
           break;
-        case BYTES_VAR_SORTED:
-        case BYTES_FIXED_SORTED:
-        case BYTES_VAR_DEREF:
-        case BYTES_FIXED_DEREF:
         default:
           assertNotNull("expected none null - " + msg, br);
-          assertEquals(0, br.length);
+          assertEquals(byteIndexValue + "", 0, br.length);
           // make sure we advance at least until base
-          ValuesEnum bytesEnum = getValuesEnum(bytesReader);
-          final int advancedTo = bytesEnum.advance(0);
-          assertTrue(byteIndexValue.name() + " advanced failed base:" + base
-              + " advancedTo: " + advancedTo, base <= advancedTo);
         }
       }
 
-      ValuesEnum bytesEnum = getValuesEnum(bytesReader);
-      final BytesRef enumRef = bytesEnum.bytes();
       // test the actual doc values added in this iteration
       assertEquals(base + numRemainingValues, r.numDocs());
       int v = 0;
@@ -393,17 +393,8 @@ public class TestDocValuesIndexing exten
           upto += bytesSize;
         }
         BytesRef br = bytes.getBytes(i, new BytesRef());
-        if (bytesEnum.docID() != i) {
-          assertEquals("seek failed for index " + i + " " + msg, i, bytesEnum
-              .advance(i));
-        }
         assertTrue(msg, br.length > 0);
         for (int j = 0; j < br.length; j++, upto++) {
-          assertTrue(" enumRef not initialized " + msg,
-              enumRef.bytes.length > 0);
-          assertEquals(
-              "EnumRef Byte at index " + j + " doesn't match - " + msg, upto,
-              enumRef.bytes[enumRef.offset + j]);
           if (!(br.bytes.length > br.offset + j))
             br = bytes.getBytes(i, new BytesRef());
           assertTrue("BytesRef index exceeded [" + msg + "] offset: "
@@ -446,33 +437,23 @@ public class TestDocValuesIndexing exten
   }
 
   private Source getSource(IndexDocValues values) throws IOException {
-    Source source;
-    if (random.nextInt(10) == 0) {
-      source = values.load();
-    } else {
-      // getSource uses cache internally
-      source = values.getSource();
+    // getSource uses cache internally
+    switch(random.nextInt(5)) {
+    case 3:
+      return values.load();
+    case 2:
+      return values.getDirectSource();
+    case 1:
+      return values.getSource();
+    default:
+      return values.getSource();
     }
-    assertNotNull(source);
-    return source;
   }
 
-  private ValuesEnum getValuesEnum(IndexDocValues values) throws IOException {
-    ValuesEnum valuesEnum;
-    if (!(values instanceof MultiIndexDocValues) && random.nextInt(10) == 0) {
-      // TODO not supported by MultiDocValues yet!
-      valuesEnum = getSource(values).getEnum();
-    } else {
-      valuesEnum = values.getEnum();
-
-    }
-    assertNotNull(valuesEnum);
-    return valuesEnum;
-  }
 
   private static EnumSet<ValueType> BYTES = EnumSet.of(ValueType.BYTES_FIXED_DEREF,
-      ValueType.BYTES_FIXED_SORTED, ValueType.BYTES_FIXED_STRAIGHT, ValueType.BYTES_VAR_DEREF,
-      ValueType.BYTES_VAR_SORTED, ValueType.BYTES_VAR_STRAIGHT);
+      ValueType.BYTES_FIXED_STRAIGHT, ValueType.BYTES_VAR_DEREF,
+      ValueType.BYTES_VAR_STRAIGHT, ValueType.BYTES_FIXED_SORTED, ValueType.BYTES_VAR_SORTED);
 
   private static EnumSet<ValueType> NUMERICS = EnumSet.of(ValueType.VAR_INTS,
       ValueType.FIXED_INTS_16, ValueType.FIXED_INTS_32,

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Mon Oct 10 19:58:24 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.search.similari
 import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
 import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
 import org.apache.lucene.search.similarities.Similarity.Stats;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TermContext;
 import org.apache.lucene.index.FieldInvertState;
@@ -364,7 +365,8 @@ final class JustCompileSearch {
     }
 
     @Override
-    public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext)
+    public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+        boolean topScorer, Bits acceptDocs)
         throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Mon Oct 10 19:58:24 2011
@@ -28,7 +28,6 @@ import org.apache.lucene.index.SlowMulti
 import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Weight.ScorerContext;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
 import org.apache.lucene.search.similarities.Similarity;
@@ -178,7 +177,8 @@ public class TestDisjunctionMaxQuery ext
     QueryUtils.check(random, dq, s);
     assertTrue(s.getTopReaderContext().isAtomic);
     final Weight dw = s.createNormalizedWeight(dq);
-    final Scorer ds = dw.scorer((AtomicReaderContext)s.getTopReaderContext(), ScorerContext.def());
+    AtomicReaderContext context = (AtomicReaderContext)s.getTopReaderContext();
+    final Scorer ds = dw.scorer(context, true, false, context.reader.getLiveDocs());
     final boolean skipOk = ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS;
     if (skipOk) {
       fail("firsttime skipTo found a match? ... "
@@ -193,7 +193,8 @@ public class TestDisjunctionMaxQuery ext
     assertTrue(s.getTopReaderContext().isAtomic);
     QueryUtils.check(random, dq, s);
     final Weight dw = s.createNormalizedWeight(dq);
-    final Scorer ds = dw.scorer((AtomicReaderContext)s.getTopReaderContext(), ScorerContext.def());
+    AtomicReaderContext context = (AtomicReaderContext)s.getTopReaderContext();
+    final Scorer ds = dw.scorer(context, true, false, context.reader.getLiveDocs());
     assertTrue("firsttime skipTo found no match",
         ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals("found wrong docid", "d4", r.document(ds.docID()).get("id"));

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTermScorer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTermScorer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestTermScorer.java Mon Oct 10 19:58:24 2011
@@ -29,7 +29,6 @@ import org.apache.lucene.index.IndexRead
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.SlowMultiReaderWrapper;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Weight.ScorerContext;
 import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
@@ -79,7 +78,8 @@ public class TestTermScorer extends Luce
     
     Weight weight = indexSearcher.createNormalizedWeight(termQuery);
     assertTrue(indexSearcher.getTopReaderContext().isAtomic);
-    Scorer ts = weight.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def().scoreDocsInOrder(true).topScorer(true));
+    AtomicReaderContext context = (AtomicReaderContext)indexSearcher.getTopReaderContext();
+    Scorer ts = weight.scorer(context, true, true, context.reader.getLiveDocs());
     // we have 2 documents with the term all in them, one document for all the
     // other values
     final List<TestHit> docs = new ArrayList<TestHit>();
@@ -140,7 +140,8 @@ public class TestTermScorer extends Luce
     
     Weight weight = indexSearcher.createNormalizedWeight(termQuery);
     assertTrue(indexSearcher.getTopReaderContext().isAtomic);
-    Scorer ts = weight.scorer((AtomicReaderContext) indexSearcher.getTopReaderContext(), ScorerContext.def().scoreDocsInOrder(true).topScorer(true));
+    AtomicReaderContext context = (AtomicReaderContext) indexSearcher.getTopReaderContext();
+    Scorer ts = weight.scorer(context, true, true, context.reader.getLiveDocs());
     assertTrue("next did not return a doc",
         ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     assertTrue("score is not correct", ts.score() == 1.6931472f);
@@ -158,8 +159,8 @@ public class TestTermScorer extends Luce
     
     Weight weight = indexSearcher.createNormalizedWeight(termQuery);
     assertTrue(indexSearcher.getTopReaderContext().isAtomic);
-
-    Scorer ts = weight.scorer((AtomicReaderContext) indexSearcher.getTopReaderContext(), ScorerContext.def().scoreDocsInOrder(true).topScorer(true));
+    AtomicReaderContext context = (AtomicReaderContext) indexSearcher.getTopReaderContext();
+    Scorer ts = weight.scorer(context, true, true, context.reader.getLiveDocs());
     assertTrue("Didn't skip", ts.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
     // The next doc should be doc 5
     assertTrue("doc should be number 5", ts.docID() == 5);

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Mon Oct 10 19:58:24 2011
@@ -23,6 +23,7 @@ import java.util.Collection;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.util.Bits;
 
 /**
  * Holds all implementations of classes in the o.a.l.s.spans package as a
@@ -82,7 +83,7 @@ final class JustCompileSearchSpans {
     }
 
     @Override
-    public Spans getSpans(AtomicReaderContext context) throws IOException {
+    public Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java Mon Oct 10 19:58:24 2011
@@ -49,7 +49,7 @@ public class MultiSpansWrapper extends S
   public static Spans wrap(ReaderContext topLevelReaderContext, SpanQuery query) throws IOException {
     AtomicReaderContext[] leaves = ReaderUtil.leaves(topLevelReaderContext);
     if(leaves.length == 1) {
-      return query.getSpans(leaves[0]);
+      return query.getSpans(leaves[0], leaves[0].reader.getLiveDocs());
     }
     return new MultiSpansWrapper(leaves, query);
   }
@@ -60,14 +60,14 @@ public class MultiSpansWrapper extends S
       return false;
     }
     if (current == null) {
-      current = query.getSpans(leaves[leafOrd]);
+      current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
     }
     while(true) {
       if (current.next()) {
         return true;
       }
       if (++leafOrd < leaves.length) {
-        current = query.getSpans(leaves[leafOrd]);
+        current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
       } else {
         current = null;
         break;
@@ -85,17 +85,17 @@ public class MultiSpansWrapper extends S
     int subIndex = ReaderUtil.subIndex(target, leaves);
     assert subIndex >= leafOrd;
     if (subIndex != leafOrd) {
-      current = query.getSpans(leaves[subIndex]);
+      current = query.getSpans(leaves[subIndex], leaves[subIndex].reader.getLiveDocs());
       leafOrd = subIndex;
     } else if (current == null) {
-      current = query.getSpans(leaves[leafOrd]);
+      current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
     }
     while (true) {
       if (current.skipTo(target - leaves[leafOrd].docBase)) {
         return true;
       }
       if (++leafOrd < leaves.length) {
-        current = query.getSpans(leaves[leafOrd]);
+        current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
       } else {
           current = null;
           break;

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Mon Oct 10 19:58:24 2011
@@ -30,7 +30,6 @@ import org.apache.lucene.search.Explanat
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight.ScorerContext;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.ReaderUtil;
@@ -170,7 +169,7 @@ public class TestNearSpansOrdered extend
     Weight w = searcher.createNormalizedWeight(q);
     ReaderContext topReaderContext = searcher.getTopReaderContext();
     AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext);
-    Scorer s = w.scorer(leaves[0], ScorerContext.def());
+    Scorer s = w.scorer(leaves[0], true, false, leaves[0].reader.getLiveDocs());
     assertEquals(1, s.advance(1));
   }
   

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Mon Oct 10 19:58:24 2011
@@ -23,7 +23,6 @@ import org.apache.lucene.search.CheckHit
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Weight.ScorerContext;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
 import org.apache.lucene.search.similarities.Similarity;
@@ -435,7 +434,7 @@ public class TestSpans extends LuceneTes
                                 slop,
                                 ordered);
   
-        spanScorer = searcher.createNormalizedWeight(snq).scorer(leaves[i], ScorerContext.def());
+        spanScorer = searcher.createNormalizedWeight(snq).scorer(leaves[i], true, false, leaves[i].reader.getLiveDocs());
       } finally {
         searcher.setSimilarityProvider(oldSim);
       }

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java Mon Oct 10 19:58:24 2011
@@ -115,12 +115,38 @@ public class TestPackedInts extends Luce
           assertEquals(fp, in.getFilePointer());
           in.close();
         }
+        
+        { // test reader iterator get
+          IndexInput in = d.openInput("out.bin", newIOContext(random));
+          PackedInts.RandomAccessReaderIterator intsEnum = PackedInts.getRandomAccessReaderIterator(in);
+          for (int i = 0; i < valueCount; i++) {
+            final String msg = "index=" + i + " ceil=" + ceil + " valueCount="
+                + valueCount + " nbits=" + nbits + " for "
+                + intsEnum.getClass().getSimpleName();
+            final int ord = random.nextInt(valueCount);
+            long seek = intsEnum.get(ord);
+            assertEquals(msg, seek, values[ord]);
+            if (random.nextBoolean() && ord < valueCount-1) {
+              if (random.nextBoolean()) {
+                assertEquals(msg, values[ord+1], intsEnum.advance(ord+1));
+              } else {
+                assertEquals(msg, values[ord+1], intsEnum.next());
+              }
+            }
+          }
+          if (intsEnum.ord() < valueCount - 1)
+            assertEquals(values[valueCount - 1], intsEnum
+                .advance(valueCount - 1));
+          assertEquals(valueCount - 1, intsEnum.ord());
+          assertEquals(fp, in.getFilePointer());
+          in.close();
+        }
         ceil *= 2;
         d.close();
       }
     }
   }
-
+  
   public void testControlledEquality() {
     final int VALUE_COUNT = 255;
     final int BITS_PER_VALUE = 8;

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java Mon Oct 10 19:58:24 2011
@@ -27,6 +27,7 @@ import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 
 public class HunspellDictionary {
 
@@ -43,11 +44,15 @@ public class HunspellDictionary {
   private static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*";
   private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s";
 
+  private static final boolean IGNORE_CASE_DEFAULT = false;
+
   private CharArrayMap<List<HunspellWord>> words;
   private CharArrayMap<List<HunspellAffix>> prefixes;
   private CharArrayMap<List<HunspellAffix>> suffixes;
 
   private FlagParsingStrategy flagParsingStrategy = new SimpleFlagParsingStrategy(); // Default flag parsing strategy
+  private boolean ignoreCase = IGNORE_CASE_DEFAULT;
+
   private final Version version;
 
   /**
@@ -61,7 +66,22 @@ public class HunspellDictionary {
    * @throws ParseException Can be thrown if the content of the files does not meet expected formats
    */
   public HunspellDictionary(InputStream affix, InputStream dictionary, Version version) throws IOException, ParseException {
-    this(affix, Arrays.asList(dictionary), version);
+    this(affix, Arrays.asList(dictionary), version, IGNORE_CASE_DEFAULT);
+  }
+
+  /**
+   * Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
+   * and dictionary files
+   *
+   * @param affix InputStream for reading the hunspell affix file
+   * @param dictionary InputStream for reading the hunspell dictionary file
+   * @param version Lucene Version
+   * @param ignoreCase If true, dictionary matching will be case insensitive
+   * @throws IOException Can be thrown while reading from the InputStreams
+   * @throws ParseException Can be thrown if the content of the files does not meet expected formats
+   */
+  public HunspellDictionary(InputStream affix, InputStream dictionary, Version version, boolean ignoreCase) throws IOException, ParseException {
+    this(affix, Arrays.asList(dictionary), version, ignoreCase);
   }
 
   /**
@@ -71,15 +91,17 @@ public class HunspellDictionary {
    * @param affix InputStream for reading the hunspell affix file
    * @param dictionaries InputStreams for reading the hunspell dictionary file
    * @param version Lucene Version
+   * @param ignoreCase If true, dictionary matching will be case insensitive
    * @throws IOException Can be thrown while reading from the InputStreams
    * @throws ParseException Can be thrown if the content of the files does not meet expected formats
    */
-  public HunspellDictionary(InputStream affix, List<InputStream> dictionaries, Version version) throws IOException, ParseException {
+  public HunspellDictionary(InputStream affix, List<InputStream> dictionaries, Version version, boolean ignoreCase) throws IOException, ParseException {
     this.version = version;
+    this.ignoreCase = ignoreCase;
     String encoding = getDictionaryEncoding(affix);
     CharsetDecoder decoder = getJavaEncoding(encoding);
     readAffixFile(affix, decoder);
-    words = new CharArrayMap<List<HunspellWord>>(version, 65535 /* guess */, false);
+    words = new CharArrayMap<List<HunspellWord>>(version, 65535 /* guess */, this.ignoreCase);
     for (InputStream dictionary : dictionaries) {
       readDictionaryFile(dictionary, decoder);
     }
@@ -129,8 +151,8 @@ public class HunspellDictionary {
    * @throws IOException Can be thrown while reading from the InputStream
    */
   private void readAffixFile(InputStream affixStream, CharsetDecoder decoder) throws IOException {
-    prefixes = new CharArrayMap<List<HunspellAffix>>(version, 8, false);
-    suffixes = new CharArrayMap<List<HunspellAffix>>(version, 8, false);
+    prefixes = new CharArrayMap<List<HunspellAffix>>(version, 8, ignoreCase);
+    suffixes = new CharArrayMap<List<HunspellAffix>>(version, 8, ignoreCase);
     
     BufferedReader reader = new BufferedReader(new InputStreamReader(affixStream, decoder));
     String line = null;
@@ -308,6 +330,9 @@ public class HunspellDictionary {
         wordForm = new HunspellWord(flagParsingStrategy.parseFlags(line.substring(flagSep + 1, end)));
         Arrays.sort(wordForm.getFlags());
         entry = line.substring(0, flagSep);
+        if(ignoreCase) {
+          entry = entry.toLowerCase(Locale.ENGLISH);
+        }
       }
       
       List<HunspellWord> entries = words.get(entry);
@@ -408,4 +433,8 @@ public class HunspellDictionary {
       return flags;
     }
   }
+
+  public boolean isIgnoreCase() {
+    return ignoreCase;
+  }
 }

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java Mon Oct 10 19:58:24 2011
@@ -21,9 +21,14 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.text.ParseException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Scanner;
 
 import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.CharacterUtils;
 import org.apache.lucene.util.Version;
 
 /**
@@ -36,6 +41,7 @@ public class HunspellStemmer {
   
   private final HunspellDictionary dictionary;
   private final StringBuilder segment = new StringBuilder();
+  private CharacterUtils charUtils = CharacterUtils.getInstance(Version.LUCENE_40);
 
   /**
    * Constructs a new HunspellStemmer which will use the provided HunspellDictionary to create its stems
@@ -79,7 +85,7 @@ public class HunspellStemmer {
    */
   public List<Stem> uniqueStems(char word[], int length) {
     List<Stem> stems = new ArrayList<Stem>();
-    CharArraySet terms = new CharArraySet(dictionary.getVersion(), 8, false);
+    CharArraySet terms = new CharArraySet(dictionary.getVersion(), 8, dictionary.isIgnoreCase());
     if (dictionary.lookupWord(word, 0, length) != null) {
       stems.add(new Stem(word, length));
       terms.add(word);
@@ -167,6 +173,12 @@ public class HunspellStemmer {
    */
   @SuppressWarnings("unchecked")
   public List<Stem> applyAffix(char strippedWord[], int length, HunspellAffix affix, int recursionDepth) {
+    if(dictionary.isIgnoreCase()) {
+      for(int i=0;i<strippedWord.length;){
+        i += Character.toChars(
+              Character.toLowerCase(charUtils.codePointAt(strippedWord, i)), strippedWord, i);
+      }
+    }
     segment.setLength(0);
     segment.append(strippedWord, 0, length);
     if (!affix.checkCondition(segment)) {
@@ -174,7 +186,7 @@ public class HunspellStemmer {
     }
 
     List<Stem> stems = new ArrayList<Stem>();
-    
+
     List<HunspellWord> words = dictionary.lookupWord(strippedWord, 0, length);
     if (words != null) {
       for (HunspellWord hunspellWord : words) {
@@ -294,15 +306,24 @@ public class HunspellStemmer {
    * @throws ParseException Can be thrown while parsing the files
    */
   public static void main(String[] args) throws IOException, ParseException {
-    if (args.length != 2) {
-      System.out.println("usage: HunspellStemmer <affix location> <dic location>");
+    boolean ignoreCase = false;
+    int offset = 0;
+    
+    if (args.length < 2) {
+      System.out.println("usage: HunspellStemmer [-i] <affix location> <dic location>");
       System.exit(1);
     }
 
-    InputStream affixInputStream = new FileInputStream(args[0]);
-    InputStream dicInputStream = new FileInputStream(args[1]);
+    if(args[offset].equals("-i")) {
+      ignoreCase = true;
+      System.out.println("Ignoring case. All stems will be returned lowercased");
+      offset++;
+    }
+    
+    InputStream affixInputStream = new FileInputStream(args[offset++]);
+    InputStream dicInputStream = new FileInputStream(args[offset++]);
 
-    HunspellDictionary dictionary = new HunspellDictionary(affixInputStream, dicInputStream, Version.LUCENE_40);
+    HunspellDictionary dictionary = new HunspellDictionary(affixInputStream, dicInputStream, Version.LUCENE_40, ignoreCase);
 
     affixInputStream.close();
     dicInputStream.close();

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Mon Oct 10 19:58:24 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 2/9/11 11:45 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 9/30/11 12:10 PM */
 
 package org.apache.lucene.analysis.standard;
 
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 2/9/11 11:45 AM from the specification file
- * <tt>C:/Users/rmuir/workspace/lucene-2911/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
+ * on 9/30/11 12:10 PM from the specification file
+ * <tt>/lucene/jflex/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
  */
 class ClassicTokenizerImpl implements StandardTokenizerInterface {
 
@@ -694,17 +694,17 @@ public final void getText(CharTermAttrib
           { return HOST;
           }
         case 13: break;
-        case 1: 
-          { /* ignore */
-          }
-        case 14: break;
         case 8: 
           { return ACRONYM_DEP;
           }
-        case 15: break;
+        case 14: break;
         case 5: 
           { return NUM;
           }
+        case 15: break;
+        case 1: 
+          { /* Break so we don't hit fall-through warning: */ break;/* ignore */
+          }
         case 16: break;
         case 9: 
           { return ACRONYM;

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex Mon Oct 10 19:58:24 2011
@@ -127,4 +127,4 @@ WHITESPACE = \r\n | [ \r\n\t\f]
 {ACRONYM_DEP}                                                  { return ACRONYM_DEP; }
 
 /** Ignore the rest */
-. | {WHITESPACE}                                               { /* ignore */ }
+. | {WHITESPACE}                                               { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro Mon Oct 10 19:58:24 2011
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-// Generated using ICU4J 4.6.0.0 on Wednesday, February 9, 2011 4:45:11 PM UTC
+// Generated using ICU4J 4.8.0.0 on Friday, September 30, 2011 4:10:42 PM UTC
 // by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
 
 

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Mon Oct 10 19:58:24 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/4/11 4:07 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 9/30/11 12:10 PM */
 
 package org.apache.lucene.analysis.standard;
 
@@ -1074,21 +1074,21 @@ public final class StandardTokenizerImpl
           { return IDEOGRAPHIC_TYPE;
           }
         case 12: break;
-        case 1: 
-          { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
-          }
-        case 13: break;
         case 8: 
           { return HANGUL_TYPE;
           }
-        case 14: break;
+        case 13: break;
         case 3: 
           { return NUMERIC_TYPE;
           }
-        case 15: break;
+        case 14: break;
         case 7: 
           { return HIRAGANA_TYPE;
           }
+        case 15: break;
+        case 1: 
+          { /* Break so we don't hit fall-through warning: */ break; /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
+          }
         case 16: break;
         default: 
           if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Mon Oct 10 19:58:24 2011
@@ -189,4 +189,4 @@ HiraganaEx = {Hiragana} ({Format} | {Ext
 //        WB3b.  ÷ (Newline | CR | LF)
 //        WB14.  Any ÷ Any
 //
-[^] { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }
+[^] { /* Break so we don't hit fall-through warning: */ break; /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java Mon Oct 10 19:58:24 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/4/11 7:48 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 9/30/11 12:10 PM */
 
 package org.apache.lucene.analysis.standard;
 
@@ -3712,25 +3712,25 @@ public final class UAX29URLEmailTokenize
           { return SOUTH_EAST_ASIAN_TYPE;
           }
         case 12: break;
+        case 1: 
+          { /* Break so we don't hit fall-through warning: */ break;/* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
+          }
+        case 13: break;
         case 10: 
           { return URL_TYPE;
           }
-        case 13: break;
+        case 14: break;
         case 9: 
           { return EMAIL_TYPE;
           }
-        case 14: break;
+        case 15: break;
         case 4: 
           { return KATAKANA_TYPE;
           }
-        case 15: break;
+        case 16: break;
         case 6: 
           { return IDEOGRAPHIC_TYPE;
           }
-        case 16: break;
-        case 1: 
-          { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
-          }
         case 17: break;
         case 8: 
           { return HANGUL_TYPE;

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex Mon Oct 10 19:58:24 2011
@@ -277,4 +277,4 @@ EMAIL = {EMAILlocalPart} "@" ({DomainNam
 //        WB3b.  ÷ (Newline | CR | LF)
 //        WB14.  Any ÷ Any
 //
-[^] { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }
+[^] { /* Break so we don't hit fall-through warning: */ break;/* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.java Mon Oct 10 19:58:24 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/4/11 4:07 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 9/30/11 12:10 PM */
 
 package org.apache.lucene.analysis.standard.std31;
 
@@ -34,8 +34,8 @@ import org.apache.lucene.analysis.tokena
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 8/4/11 4:07 PM from the specification file
- * <tt>/home/rmuir/workspace/lucene-clean-trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex</tt>
+ * on 9/30/11 12:10 PM from the specification file
+ * <tt>/lucene/jflex/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex</tt>
  */
 public final class StandardTokenizerImpl31 implements StandardTokenizerInterface {
 
@@ -1055,21 +1055,21 @@ public final class StandardTokenizerImpl
           { return IDEOGRAPHIC_TYPE;
           }
         case 12: break;
-        case 1: 
-          { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
-          }
-        case 13: break;
         case 8: 
           { return HANGUL_TYPE;
           }
-        case 14: break;
+        case 13: break;
         case 3: 
           { return NUMERIC_TYPE;
           }
-        case 15: break;
+        case 14: break;
         case 7: 
           { return HIRAGANA_TYPE;
           }
+        case 15: break;
+        case 1: 
+          { /* Break so we don't hit fall-through warning: */ break; /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
+          }
         case 16: break;
         default: 
           if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/StandardTokenizerImpl31.jflex Mon Oct 10 19:58:24 2011
@@ -181,4 +181,4 @@ ExtendNumLetEx = {ExtendNumLet}         
 //        WB3b.  ÷ (Newline | CR | LF)
 //        WB14.  Any ÷ Any
 //
-[^] { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }
+[^] { /* Break so we don't hit fall-through warning: */ break; /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.java Mon Oct 10 19:58:24 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/4/11 7:33 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 9/30/11 12:10 PM */
 
 package org.apache.lucene.analysis.standard.std31;
 
@@ -34,8 +34,8 @@ import org.apache.lucene.analysis.tokena
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 8/4/11 7:33 PM from the specification file
- * <tt>/home/rmuir/workspace/lucene-clean-trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex</tt>
+ * on 9/30/11 12:10 PM from the specification file
+ * <tt>/lucene/jflex/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex</tt>
  */
 public final class UAX29URLEmailTokenizerImpl31 implements StandardTokenizerInterface {
 
@@ -3622,25 +3622,25 @@ public final class UAX29URLEmailTokenize
           { return SOUTH_EAST_ASIAN_TYPE;
           }
         case 12: break;
+        case 1: 
+          { /* Break so we don't hit fall-through warning: */ break;/* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
+          }
+        case 13: break;
         case 10: 
           { return URL_TYPE;
           }
-        case 13: break;
+        case 14: break;
         case 9: 
           { return EMAIL_TYPE;
           }
-        case 14: break;
+        case 15: break;
         case 4: 
           { return KATAKANA_TYPE;
           }
-        case 15: break;
+        case 16: break;
         case 6: 
           { return IDEOGRAPHIC_TYPE;
           }
-        case 16: break;
-        case 1: 
-          { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */
-          }
         case 17: break;
         case 8: 
           { return HANGUL_TYPE;

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex Mon Oct 10 19:58:24 2011
@@ -266,4 +266,4 @@ EMAIL = {EMAILlocalPart} "@" ({DomainNam
 //        WB3b.  ÷ (Newline | CR | LF)
 //        WB14.  Any ÷ Any
 //
-[^] { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }
+[^] { /* Break so we don't hit fall-through warning: */ break;/* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }

Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java Mon Oct 10 19:58:24 2011
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 2/9/11 11:45 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 9/30/11 12:11 PM */
 
 package org.apache.lucene.analysis.wikipedia;
 
@@ -25,8 +25,8 @@ import org.apache.lucene.analysis.tokena
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 2/9/11 11:45 AM from the specification file
- * <tt>C:/Users/rmuir/workspace/lucene-2911/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
+ * on 9/30/11 12:11 PM from the specification file
+ * <tt>/lucene/jflex/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
  */
 class WikipediaTokenizerImpl {
 
@@ -92,15 +92,16 @@ class WikipediaTokenizerImpl {
     "\1\13\1\14\1\10\1\15\1\16\1\15\1\17\1\20"+
     "\1\10\1\21\1\10\4\22\1\23\1\22\1\24\1\25"+
     "\1\26\3\0\1\27\14\0\1\30\1\31\1\32\1\33"+
-    "\1\11\1\0\1\34\1\35\1\0\1\36\1\0\1\37"+
-    "\3\0\1\40\1\41\2\42\1\41\2\43\2\0\1\42"+
-    "\1\0\14\42\1\41\3\0\1\11\1\44\3\0\1\45"+
-    "\1\46\5\0\1\47\4\0\1\47\2\0\2\47\2\0"+
-    "\1\11\5\0\1\31\1\41\1\42\1\50\3\0\1\11"+
-    "\2\0\1\51\30\0\1\52\2\0\1\53\1\54\1\55";
+    "\1\11\1\0\1\34\1\35\1\36\1\0\1\37\1\0"+
+    "\1\40\3\0\1\41\1\42\2\43\1\42\2\44\2\0"+
+    "\1\43\1\0\14\43\1\42\3\0\1\11\1\45\3\0"+
+    "\1\46\1\47\5\0\1\50\4\0\1\50\2\0\2\50"+
+    "\2\0\1\11\5\0\1\31\1\42\1\43\1\51\3\0"+
+    "\1\11\2\0\1\52\30\0\1\53\2\0\1\54\1\55"+
+    "\1\56";
 
   private static int [] zzUnpackAction() {
-    int [] result = new int[183];
+    int [] result = new int[184];
     int offset = 0;
     offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
     return result;
@@ -134,23 +135,23 @@ class WikipediaTokenizerImpl {
     "\0\u070c\0\u0738\0\u0764\0\u0790\0\u01b8\0\u01b8\0\u07bc\0\u07e8"+
     "\0\u0814\0\u01b8\0\u0840\0\u086c\0\u0898\0\u08c4\0\u08f0\0\u091c"+
     "\0\u0948\0\u0974\0\u09a0\0\u09cc\0\u09f8\0\u0a24\0\u0a50\0\u0a7c"+
-    "\0\u01b8\0\u01b8\0\u0aa8\0\u0ad4\0\u0b00\0\u0b00\0\u0b2c\0\u0b58"+
-    "\0\u0b84\0\u0bb0\0\u0bdc\0\u0c08\0\u0c34\0\u0c60\0\u0c8c\0\u0cb8"+
-    "\0\u0ce4\0\u0d10\0\u0898\0\u0d3c\0\u0d68\0\u0d94\0\u0dc0\0\u0dec"+
-    "\0\u0e18\0\u0e44\0\u0e70\0\u0e9c\0\u0ec8\0\u0ef4\0\u0f20\0\u0f4c"+
-    "\0\u0f78\0\u0fa4\0\u0fd0\0\u0ffc\0\u1028\0\u1054\0\u1080\0\u10ac"+
-    "\0\u10d8\0\u01b8\0\u1104\0\u1130\0\u115c\0\u1188\0\u01b8\0\u11b4"+
-    "\0\u11e0\0\u120c\0\u1238\0\u1264\0\u1290\0\u12bc\0\u12e8\0\u1314"+
-    "\0\u1340\0\u136c\0\u1398\0\u13c4\0\u086c\0\u09f8\0\u13f0\0\u141c"+
-    "\0\u1448\0\u1474\0\u14a0\0\u14cc\0\u14f8\0\u1524\0\u01b8\0\u1550"+
-    "\0\u157c\0\u15a8\0\u15d4\0\u1600\0\u162c\0\u1658\0\u1684\0\u16b0"+
-    "\0\u01b8\0\u16dc\0\u1708\0\u1734\0\u1760\0\u178c\0\u17b8\0\u17e4"+
-    "\0\u1810\0\u183c\0\u1868\0\u1894\0\u18c0\0\u18ec\0\u1918\0\u1944"+
-    "\0\u1970\0\u199c\0\u19c8\0\u19f4\0\u1a20\0\u1a4c\0\u1a78\0\u1aa4"+
-    "\0\u1ad0\0\u1afc\0\u1b28\0\u1b54\0\u01b8\0\u01b8\0\u01b8";
+    "\0\u01b8\0\u01b8\0\u0aa8\0\u0ad4\0\u0b00\0\u0b00\0\u01b8\0\u0b2c"+
+    "\0\u0b58\0\u0b84\0\u0bb0\0\u0bdc\0\u0c08\0\u0c34\0\u0c60\0\u0c8c"+
+    "\0\u0cb8\0\u0ce4\0\u0d10\0\u0898\0\u0d3c\0\u0d68\0\u0d94\0\u0dc0"+
+    "\0\u0dec\0\u0e18\0\u0e44\0\u0e70\0\u0e9c\0\u0ec8\0\u0ef4\0\u0f20"+
+    "\0\u0f4c\0\u0f78\0\u0fa4\0\u0fd0\0\u0ffc\0\u1028\0\u1054\0\u1080"+
+    "\0\u10ac\0\u10d8\0\u01b8\0\u1104\0\u1130\0\u115c\0\u1188\0\u01b8"+
+    "\0\u11b4\0\u11e0\0\u120c\0\u1238\0\u1264\0\u1290\0\u12bc\0\u12e8"+
+    "\0\u1314\0\u1340\0\u136c\0\u1398\0\u13c4\0\u086c\0\u09f8\0\u13f0"+
+    "\0\u141c\0\u1448\0\u1474\0\u14a0\0\u14cc\0\u14f8\0\u1524\0\u01b8"+
+    "\0\u1550\0\u157c\0\u15a8\0\u15d4\0\u1600\0\u162c\0\u1658\0\u1684"+
+    "\0\u16b0\0\u01b8\0\u16dc\0\u1708\0\u1734\0\u1760\0\u178c\0\u17b8"+
+    "\0\u17e4\0\u1810\0\u183c\0\u1868\0\u1894\0\u18c0\0\u18ec\0\u1918"+
+    "\0\u1944\0\u1970\0\u199c\0\u19c8\0\u19f4\0\u1a20\0\u1a4c\0\u1a78"+
+    "\0\u1aa4\0\u1ad0\0\u1afc\0\u1b28\0\u1b54\0\u01b8\0\u01b8\0\u01b8";
 
   private static int [] zzUnpackRowMap() {
-    int [] result = new int[183];
+    int [] result = new int[184];
     int offset = 0;
     offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
     return result;
@@ -208,115 +209,115 @@ class WikipediaTokenizerImpl {
     "\3\0\3\40\16\0\4\40\7\0\2\40\1\113\12\40"+
     "\3\0\3\40\2\0\1\114\67\0\4\44\7\0\15\44"+
     "\3\0\3\44\24\0\1\36\55\0\1\115\43\0\4\47"+
-    "\7\0\15\47\3\0\3\47\26\0\1\116\37\0\1\111"+
-    "\57\0\4\52\7\0\15\52\3\0\3\52\11\0\1\117"+
+    "\7\0\15\47\3\0\3\47\26\0\1\116\37\0\1\117"+
+    "\57\0\4\52\7\0\15\52\3\0\3\52\11\0\1\120"+
     "\4\0\4\70\7\0\15\70\3\0\3\70\16\0\4\54"+
-    "\7\0\15\54\3\0\3\54\47\0\1\111\6\0\1\120"+
-    "\63\0\1\121\57\0\4\62\7\0\15\62\3\0\3\62"+
-    "\24\0\1\56\55\0\1\122\43\0\4\70\7\0\15\70"+
-    "\3\0\3\70\14\0\1\36\1\0\4\123\1\0\3\124"+
-    "\3\0\15\123\3\0\3\123\14\0\1\36\1\0\4\123"+
-    "\1\0\3\124\3\0\3\123\1\125\11\123\3\0\3\123"+
-    "\16\0\1\126\1\0\1\126\10\0\15\126\3\0\3\126"+
-    "\16\0\1\127\1\130\1\131\1\132\7\0\15\127\3\0"+
-    "\3\127\16\0\1\133\1\0\1\133\10\0\15\133\3\0"+
-    "\3\133\16\0\1\134\1\135\1\134\1\135\7\0\15\134"+
-    "\3\0\3\134\16\0\1\136\2\137\1\140\7\0\15\136"+
-    "\3\0\3\136\16\0\1\100\2\141\10\0\15\100\3\0"+
-    "\3\100\16\0\1\142\2\143\1\144\7\0\15\142\3\0"+
-    "\3\142\16\0\4\135\7\0\15\135\3\0\3\135\16\0"+
-    "\1\145\2\146\1\147\7\0\15\145\3\0\3\145\16\0"+
-    "\1\150\2\151\1\152\7\0\15\150\3\0\3\150\16\0"+
-    "\1\153\1\143\1\154\1\144\7\0\15\153\3\0\3\153"+
-    "\16\0\1\155\2\130\1\132\7\0\15\155\3\0\3\155"+
-    "\30\0\1\156\1\157\64\0\1\160\27\0\4\40\7\0"+
-    "\2\40\1\161\12\40\3\0\3\40\2\0\1\162\101\0"+
-    "\1\163\1\164\40\0\4\70\7\0\6\70\1\165\6\70"+
-    "\3\0\3\70\2\0\1\166\63\0\1\167\71\0\1\170"+
-    "\1\171\34\0\1\172\1\0\1\36\1\0\4\123\1\0"+
-    "\3\124\3\0\15\123\3\0\3\123\16\0\4\173\1\0"+
-    "\3\124\3\0\15\173\3\0\3\173\12\0\1\172\1\0"+
-    "\1\36\1\0\4\123\1\0\3\124\3\0\10\123\1\174"+
-    "\4\123\3\0\3\123\2\0\1\73\13\0\1\126\1\0"+
-    "\1\126\10\0\15\126\3\0\3\126\3\0\1\175\1\0"+
-    "\1\102\2\176\6\0\1\127\1\130\1\131\1\132\7\0"+
-    "\15\127\3\0\3\127\3\0\1\177\1\0\1\102\2\200"+
-    "\1\0\1\201\3\0\1\201\3\130\1\132\7\0\15\130"+
-    "\3\0\3\130\3\0\1\202\1\0\1\102\2\200\1\0"+
-    "\1\201\3\0\1\201\1\131\1\130\1\131\1\132\7\0"+
-    "\15\131\3\0\3\131\3\0\1\203\1\0\1\102\2\176"+
-    "\6\0\4\132\7\0\15\132\3\0\3\132\3\0\1\204"+
-    "\2\0\1\204\7\0\1\134\1\135\1\134\1\135\7\0"+
-    "\15\134\3\0\3\134\3\0\1\204\2\0\1\204\7\0"+
-    "\4\135\7\0\15\135\3\0\3\135\3\0\1\176\1\0"+
-    "\1\102\2\176\6\0\1\136\2\137\1\140\7\0\15\136"+
-    "\3\0\3\136\3\0\1\200\1\0\1\102\2\200\1\0"+
-    "\1\201\3\0\1\201\3\137\1\140\7\0\15\137\3\0"+
-    "\3\137\3\0\1\176\1\0\1\102\2\176\6\0\4\140"+
-    "\7\0\15\140\3\0\3\140\3\0\1\201\2\0\2\201"+
-    "\1\0\1\201\3\0\1\201\3\141\10\0\15\141\3\0"+
-    "\3\141\3\0\1\106\1\0\1\102\2\77\1\0\1\100"+
-    "\3\0\1\100\1\142\2\143\1\144\7\0\15\142\3\0"+
-    "\3\142\3\0\1\101\1\0\1\102\2\103\1\0\1\104"+
-    "\3\0\1\104\3\143\1\144\7\0\15\143\3\0\3\143"+
+    "\7\0\15\54\3\0\3\54\47\0\1\117\6\0\1\121"+
+    "\63\0\1\122\57\0\4\62\7\0\15\62\3\0\3\62"+
+    "\24\0\1\56\55\0\1\123\43\0\4\70\7\0\15\70"+
+    "\3\0\3\70\14\0\1\36\1\0\4\124\1\0\3\125"+
+    "\3\0\15\124\3\0\3\124\14\0\1\36\1\0\4\124"+
+    "\1\0\3\125\3\0\3\124\1\126\11\124\3\0\3\124"+
+    "\16\0\1\127\1\0\1\127\10\0\15\127\3\0\3\127"+
+    "\16\0\1\130\1\131\1\132\1\133\7\0\15\130\3\0"+
+    "\3\130\16\0\1\134\1\0\1\134\10\0\15\134\3\0"+
+    "\3\134\16\0\1\135\1\136\1\135\1\136\7\0\15\135"+
+    "\3\0\3\135\16\0\1\137\2\140\1\141\7\0\15\137"+
+    "\3\0\3\137\16\0\1\100\2\142\10\0\15\100\3\0"+
+    "\3\100\16\0\1\143\2\144\1\145\7\0\15\143\3\0"+
+    "\3\143\16\0\4\136\7\0\15\136\3\0\3\136\16\0"+
+    "\1\146\2\147\1\150\7\0\15\146\3\0\3\146\16\0"+
+    "\1\151\2\152\1\153\7\0\15\151\3\0\3\151\16\0"+
+    "\1\154\1\144\1\155\1\145\7\0\15\154\3\0\3\154"+
+    "\16\0\1\156\2\131\1\133\7\0\15\156\3\0\3\156"+
+    "\30\0\1\157\1\160\64\0\1\161\27\0\4\40\7\0"+
+    "\2\40\1\162\12\40\3\0\3\40\2\0\1\163\101\0"+
+    "\1\164\1\165\40\0\4\70\7\0\6\70\1\166\6\70"+
+    "\3\0\3\70\2\0\1\167\63\0\1\170\71\0\1\171"+
+    "\1\172\34\0\1\173\1\0\1\36\1\0\4\124\1\0"+
+    "\3\125\3\0\15\124\3\0\3\124\16\0\4\174\1\0"+
+    "\3\125\3\0\15\174\3\0\3\174\12\0\1\173\1\0"+
+    "\1\36\1\0\4\124\1\0\3\125\3\0\10\124\1\175"+
+    "\4\124\3\0\3\124\2\0\1\73\13\0\1\127\1\0"+
+    "\1\127\10\0\15\127\3\0\3\127\3\0\1\176\1\0"+
+    "\1\102\2\177\6\0\1\130\1\131\1\132\1\133\7\0"+
+    "\15\130\3\0\3\130\3\0\1\200\1\0\1\102\2\201"+
+    "\1\0\1\202\3\0\1\202\3\131\1\133\7\0\15\131"+
+    "\3\0\3\131\3\0\1\203\1\0\1\102\2\201\1\0"+
+    "\1\202\3\0\1\202\1\132\1\131\1\132\1\133\7\0"+
+    "\15\132\3\0\3\132\3\0\1\204\1\0\1\102\2\177"+
+    "\6\0\4\133\7\0\15\133\3\0\3\133\3\0\1\205"+
+    "\2\0\1\205\7\0\1\135\1\136\1\135\1\136\7\0"+
+    "\15\135\3\0\3\135\3\0\1\205\2\0\1\205\7\0"+
+    "\4\136\7\0\15\136\3\0\3\136\3\0\1\177\1\0"+
+    "\1\102\2\177\6\0\1\137\2\140\1\141\7\0\15\137"+
+    "\3\0\3\137\3\0\1\201\1\0\1\102\2\201\1\0"+
+    "\1\202\3\0\1\202\3\140\1\141\7\0\15\140\3\0"+
+    "\3\140\3\0\1\177\1\0\1\102\2\177\6\0\4\141"+
+    "\7\0\15\141\3\0\3\141\3\0\1\202\2\0\2\202"+
+    "\1\0\1\202\3\0\1\202\3\142\10\0\15\142\3\0"+
+    "\3\142\3\0\1\106\1\0\1\102\2\77\1\0\1\100"+
+    "\3\0\1\100\1\143\2\144\1\145\7\0\15\143\3\0"+
+    "\3\143\3\0\1\101\1\0\1\102\2\103\1\0\1\104"+
+    "\3\0\1\104\3\144\1\145\7\0\15\144\3\0\3\144"+
     "\3\0\1\106\1\0\1\102\2\77\1\0\1\100\3\0"+
-    "\1\100\4\144\7\0\15\144\3\0\3\144\3\0\1\77"+
-    "\1\0\1\102\2\77\1\0\1\100\3\0\1\100\1\145"+
-    "\2\146\1\147\7\0\15\145\3\0\3\145\3\0\1\103"+
-    "\1\0\1\102\2\103\1\0\1\104\3\0\1\104\3\146"+
-    "\1\147\7\0\15\146\3\0\3\146\3\0\1\77\1\0"+
-    "\1\102\2\77\1\0\1\100\3\0\1\100\4\147\7\0"+
-    "\15\147\3\0\3\147\3\0\1\100\2\0\2\100\1\0"+
-    "\1\100\3\0\1\100\1\150\2\151\1\152\7\0\15\150"+
-    "\3\0\3\150\3\0\1\104\2\0\2\104\1\0\1\104"+
-    "\3\0\1\104\3\151\1\152\7\0\15\151\3\0\3\151"+
+    "\1\100\4\145\7\0\15\145\3\0\3\145\3\0\1\77"+
+    "\1\0\1\102\2\77\1\0\1\100\3\0\1\100\1\146"+
+    "\2\147\1\150\7\0\15\146\3\0\3\146\3\0\1\103"+
+    "\1\0\1\102\2\103\1\0\1\104\3\0\1\104\3\147"+
+    "\1\150\7\0\15\147\3\0\3\147\3\0\1\77\1\0"+
+    "\1\102\2\77\1\0\1\100\3\0\1\100\4\150\7\0"+
+    "\15\150\3\0\3\150\3\0\1\100\2\0\2\100\1\0"+
+    "\1\100\3\0\1\100\1\151\2\152\1\153\7\0\15\151"+
+    "\3\0\3\151\3\0\1\104\2\0\2\104\1\0\1\104"+
+    "\3\0\1\104\3\152\1\153\7\0\15\152\3\0\3\152"+
     "\3\0\1\100\2\0\2\100\1\0\1\100\3\0\1\100"+
-    "\4\152\7\0\15\152\3\0\3\152\3\0\1\205\1\0"+
-    "\1\102\2\77\1\0\1\100\3\0\1\100\1\153\1\143"+
-    "\1\154\1\144\7\0\15\153\3\0\3\153\3\0\1\206"+
-    "\1\0\1\102\2\103\1\0\1\104\3\0\1\104\1\154"+
-    "\1\143\1\154\1\144\7\0\15\154\3\0\3\154\3\0"+
-    "\1\203\1\0\1\102\2\176\6\0\1\155\2\130\1\132"+
-    "\7\0\15\155\3\0\3\155\31\0\1\157\54\0\1\207"+
-    "\64\0\1\210\26\0\4\40\7\0\15\40\3\0\1\40"+
-    "\1\211\1\40\31\0\1\164\54\0\1\212\35\0\1\36"+
-    "\1\0\4\123\1\0\3\124\3\0\3\123\1\213\11\123"+
-    "\3\0\3\123\2\0\1\214\102\0\1\171\54\0\1\215"+
-    "\34\0\1\216\52\0\1\172\3\0\4\173\7\0\15\173"+
-    "\3\0\3\173\12\0\1\172\1\0\1\217\1\0\4\123"+
-    "\1\0\3\124\3\0\15\123\3\0\3\123\16\0\1\220"+
-    "\1\132\1\220\1\132\7\0\15\220\3\0\3\220\16\0"+
-    "\4\140\7\0\15\140\3\0\3\140\16\0\4\144\7\0"+
-    "\15\144\3\0\3\144\16\0\4\147\7\0\15\147\3\0"+
-    "\3\147\16\0\4\152\7\0\15\152\3\0\3\152\16\0"+
-    "\1\221\1\144\1\221\1\144\7\0\15\221\3\0\3\221"+
-    "\16\0\4\132\7\0\15\132\3\0\3\132\16\0\4\222"+
-    "\7\0\15\222\3\0\3\222\33\0\1\223\61\0\1\224"+
-    "\30\0\4\40\6\0\1\225\15\40\3\0\2\40\1\226"+
-    "\33\0\1\227\32\0\1\172\1\0\1\36\1\0\4\123"+
-    "\1\0\3\124\3\0\10\123\1\230\4\123\3\0\3\123"+
-    "\2\0\1\231\104\0\1\232\36\0\4\233\7\0\15\233"+
-    "\3\0\3\233\3\0\1\175\1\0\1\102\2\176\6\0"+
-    "\1\220\1\132\1\220\1\132\7\0\15\220\3\0\3\220"+
-    "\3\0\1\205\1\0\1\102\2\77\1\0\1\100\3\0"+
-    "\1\100\1\221\1\144\1\221\1\144\7\0\15\221\3\0"+
-    "\3\221\3\0\1\204\2\0\1\204\7\0\4\222\7\0"+
-    "\15\222\3\0\3\222\34\0\1\234\55\0\1\235\26\0"+
-    "\1\236\60\0\4\40\6\0\1\225\15\40\3\0\3\40"+
-    "\34\0\1\237\31\0\1\172\1\0\1\111\1\0\4\123"+
-    "\1\0\3\124\3\0\15\123\3\0\3\123\34\0\1\240"+
-    "\32\0\1\241\2\0\4\233\7\0\15\233\3\0\3\233"+
-    "\35\0\1\242\62\0\1\243\20\0\1\244\77\0\1\245"+
-    "\53\0\1\246\32\0\1\36\1\0\4\173\1\0\3\124"+
-    "\3\0\15\173\3\0\3\173\36\0\1\247\53\0\1\250"+
-    "\33\0\4\251\7\0\15\251\3\0\3\251\36\0\1\252"+
-    "\53\0\1\253\54\0\1\254\61\0\1\255\11\0\1\256"+
-    "\12\0\4\251\7\0\15\251\3\0\3\251\37\0\1\257"+
-    "\53\0\1\260\54\0\1\261\22\0\1\13\62\0\4\262"+
-    "\7\0\15\262\3\0\3\262\40\0\1\263\53\0\1\264"+
-    "\43\0\1\265\26\0\2\262\1\0\2\262\1\0\2\262"+
-    "\2\0\5\262\7\0\15\262\3\0\4\262\27\0\1\266"+
-    "\53\0\1\267\24\0";
+    "\4\153\7\0\15\153\3\0\3\153\3\0\1\206\1\0"+
+    "\1\102\2\77\1\0\1\100\3\0\1\100\1\154\1\144"+
+    "\1\155\1\145\7\0\15\154\3\0\3\154\3\0\1\207"+
+    "\1\0\1\102\2\103\1\0\1\104\3\0\1\104\1\155"+
+    "\1\144\1\155\1\145\7\0\15\155\3\0\3\155\3\0"+
+    "\1\204\1\0\1\102\2\177\6\0\1\156\2\131\1\133"+
+    "\7\0\15\156\3\0\3\156\31\0\1\160\54\0\1\210"+
+    "\64\0\1\211\26\0\4\40\7\0\15\40\3\0\1\40"+
+    "\1\212\1\40\31\0\1\165\54\0\1\213\35\0\1\36"+
+    "\1\0\4\124\1\0\3\125\3\0\3\124\1\214\11\124"+
+    "\3\0\3\124\2\0\1\215\102\0\1\172\54\0\1\216"+
+    "\34\0\1\217\52\0\1\173\3\0\4\174\7\0\15\174"+
+    "\3\0\3\174\12\0\1\173\1\0\1\220\1\0\4\124"+
+    "\1\0\3\125\3\0\15\124\3\0\3\124\16\0\1\221"+
+    "\1\133\1\221\1\133\7\0\15\221\3\0\3\221\16\0"+
+    "\4\141\7\0\15\141\3\0\3\141\16\0\4\145\7\0"+
+    "\15\145\3\0\3\145\16\0\4\150\7\0\15\150\3\0"+
+    "\3\150\16\0\4\153\7\0\15\153\3\0\3\153\16\0"+
+    "\1\222\1\145\1\222\1\145\7\0\15\222\3\0\3\222"+
+    "\16\0\4\133\7\0\15\133\3\0\3\133\16\0\4\223"+
+    "\7\0\15\223\3\0\3\223\33\0\1\224\61\0\1\225"+
+    "\30\0\4\40\6\0\1\226\15\40\3\0\2\40\1\227"+
+    "\33\0\1\230\32\0\1\173\1\0\1\36\1\0\4\124"+
+    "\1\0\3\125\3\0\10\124\1\231\4\124\3\0\3\124"+
+    "\2\0\1\232\104\0\1\233\36\0\4\234\7\0\15\234"+
+    "\3\0\3\234\3\0\1\176\1\0\1\102\2\177\6\0"+
+    "\1\221\1\133\1\221\1\133\7\0\15\221\3\0\3\221"+
+    "\3\0\1\206\1\0\1\102\2\77\1\0\1\100\3\0"+
+    "\1\100\1\222\1\145\1\222\1\145\7\0\15\222\3\0"+
+    "\3\222\3\0\1\205\2\0\1\205\7\0\4\223\7\0"+
+    "\15\223\3\0\3\223\34\0\1\235\55\0\1\236\26\0"+
+    "\1\237\60\0\4\40\6\0\1\226\15\40\3\0\3\40"+
+    "\34\0\1\240\31\0\1\173\1\0\1\117\1\0\4\124"+
+    "\1\0\3\125\3\0\15\124\3\0\3\124\34\0\1\241"+
+    "\32\0\1\242\2\0\4\234\7\0\15\234\3\0\3\234"+
+    "\35\0\1\243\62\0\1\244\20\0\1\245\77\0\1\246"+
+    "\53\0\1\247\32\0\1\36\1\0\4\174\1\0\3\125"+
+    "\3\0\15\174\3\0\3\174\36\0\1\250\53\0\1\251"+
+    "\33\0\4\252\7\0\15\252\3\0\3\252\36\0\1\253"+
+    "\53\0\1\254\54\0\1\255\61\0\1\256\11\0\1\257"+
+    "\12\0\4\252\7\0\15\252\3\0\3\252\37\0\1\260"+
+    "\53\0\1\261\54\0\1\262\22\0\1\13\62\0\4\263"+
+    "\7\0\15\263\3\0\3\263\40\0\1\264\53\0\1\265"+
+    "\43\0\1\266\26\0\2\263\1\0\2\263\1\0\2\263"+
+    "\2\0\5\263\7\0\15\263\3\0\4\263\27\0\1\267"+
+    "\53\0\1\270\24\0";
 
   private static int [] zzUnpackTrans() {
     int [] result = new int[7040];
@@ -359,15 +360,15 @@ class WikipediaTokenizerImpl {
   private static final String ZZ_ATTRIBUTE_PACKED_0 =
     "\12\0\1\11\7\1\1\11\3\1\1\11\6\1\1\11"+
     "\2\1\1\11\14\1\1\11\6\1\2\11\3\0\1\11"+
-    "\14\0\2\1\2\11\1\1\1\0\2\1\1\0\1\1"+
-    "\1\0\1\1\3\0\7\1\2\0\1\1\1\0\15\1"+
-    "\3\0\1\1\1\11\3\0\1\1\1\11\5\0\1\1"+
-    "\4\0\1\1\2\0\2\1\2\0\1\1\5\0\1\11"+
-    "\3\1\3\0\1\1\2\0\1\11\30\0\1\1\2\0"+
-    "\3\11";
+    "\14\0\2\1\2\11\1\1\1\0\2\1\1\11\1\0"+
+    "\1\1\1\0\1\1\3\0\7\1\2\0\1\1\1\0"+
+    "\15\1\3\0\1\1\1\11\3\0\1\1\1\11\5\0"+
+    "\1\1\4\0\1\1\2\0\2\1\2\0\1\1\5\0"+
+    "\1\11\3\1\3\0\1\1\2\0\1\11\30\0\1\1"+
+    "\2\0\3\11";
 
   private static int [] zzUnpackAttribute() {
-    int [] result = new int[183];
+    int [] result = new int[184];
     int offset = 0;
     offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
     return result;
@@ -809,186 +810,190 @@ final int setText(StringBuilder buffer){
       zzMarkedPos = zzMarkedPosL;
 
       switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
-        case 16: 
-          { currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
-          }
-        case 46: break;
-        case 39: 
-          { positionInc = 1; return ACRONYM;
+        case 44: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);/* Break so we don't hit fall-through warning: */ break;
           }
         case 47: break;
-        case 8: 
-          { /* ignore */
+        case 37: 
+          { currentTokType = BOLD_ITALICS;  yybegin(FIVE_SINGLE_QUOTES_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 48: break;
-        case 20: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);
+        case 16: 
+          { currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 49: break;
-        case 35: 
-          { positionInc = 1; return COMPANY;
+        case 20: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 50: break;
-        case 4: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
+        case 40: 
+          { positionInc = 1; return ACRONYM;
           }
         case 51: break;
-        case 25: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
+        case 5: 
+          { positionInc = 1; /* Break so we don't hit fall-through warning: */ break;
           }
         case 52: break;
-        case 43: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
+        case 36: 
+          { positionInc = 1; return COMPANY;
           }
         case 53: break;
-        case 22: 
-          { numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
+        case 10: 
+          { numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;
           }
         case 54: break;
-        case 34: 
-          { positionInc = 1; return NUM;
+        case 15: 
+          { currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING); /* Break so we don't hit fall-through warning: */ break;
           }
         case 55: break;
-        case 32: 
-          { positionInc = 1; return APOSTROPHE;
+        case 22: 
+          { numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}/* Break so we don't hit fall-through warning: */ break;
           }
         case 56: break;
-        case 23: 
-          { numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
+        case 35: 
+          { positionInc = 1; return NUM;
           }
         case 57: break;
-        case 21: 
-          { yybegin(STRING); return currentTokType;/*pipe*/
+        case 33: 
+          { positionInc = 1; return APOSTROPHE;
           }
         case 58: break;
-        case 2: 
-          { positionInc = 1; return ALPHANUM;
+        case 21: 
+          { yybegin(STRING); return currentTokType;/*pipe*/
           }
         case 59: break;
-        case 29: 
-          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0;  yybegin(INTERNAL_LINK_STATE);
+        case 18: 
+          { /* Break so we don't hit fall-through warning: */ break;/* ignore STRING */
           }
         case 60: break;
-        case 17: 
-          { yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
+        case 2: 
+          { positionInc = 1; return ALPHANUM;
           }
         case 61: break;
-        case 44: 
-          { currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
+        case 1: 
+          { numWikiTokensSeen = 0;  positionInc = 1; /* Break so we don't hit fall-through warning: */ break;
           }
         case 62: break;
-        case 26: 
-          { yybegin(YYINITIAL);
+        case 17: 
+          { yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
           }
         case 63: break;
-        case 3: 
-          { positionInc = 1; return CJ;
+        case 39: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end sub header*/
           }
         case 64: break;
-        case 38: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
+        case 29: 
+          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0;  yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 65: break;
-        case 15: 
-          { currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
+        case 46: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 66: break;
-        case 30: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
+        case 27: 
+          { numLinkToks = 0; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;
           }
         case 67: break;
-        case 6: 
-          { yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
+        case 4: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);/* Break so we don't hit fall-through warning: */ break;
           }
         case 68: break;
-        case 5: 
-          { positionInc = 1;
+        case 38: 
+          { numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end bold*/
           }
         case 69: break;
-        case 19: 
-          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
+        case 13: 
+          { currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 70: break;
-        case 42: 
-          { positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
+        case 3: 
+          { positionInc = 1; return CJ;
           }
         case 71: break;
-        case 27: 
-          { numLinkToks = 0; yybegin(YYINITIAL);
+        case 45: 
+          { currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 72: break;
-        case 11: 
-          { currentTokType = BOLD;  yybegin(THREE_SINGLE_QUOTES_STATE);
+        case 6: 
+          { yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 73: break;
-        case 13: 
-          { currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
+        case 11: 
+          { currentTokType = BOLD;  yybegin(THREE_SINGLE_QUOTES_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 74: break;
-        case 14: 
-          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;
+        case 25: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);/* Break so we don't hit fall-through warning: */ break;
           }
         case 75: break;
-        case 45: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
+        case 8: 
+          { /* Break so we don't hit fall-through warning: */ break;/* ignore */
           }
         case 76: break;
-        case 28: 
-          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
+        case 19: 
+          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
           }
         case 77: break;
-        case 37: 
-          { numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
+        case 43: 
+          { positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
           }
         case 78: break;
-        case 9: 
-          { if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
+        case 42: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end bold italics*/
           }
         case 79: break;
-        case 7: 
-          { yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
+        case 30: 
+          { yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;
           }
         case 80: break;
-        case 24: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
+        case 14: 
+          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;
           }
         case 81: break;
-        case 40: 
-          { positionInc = 1; return EMAIL;
+        case 9: 
+          { if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
           }
         case 82: break;
-        case 1: 
-          { numWikiTokensSeen = 0;  positionInc = 1;
+        case 7: 
+          { yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 83: break;
-        case 18: 
-          { /* ignore STRING */
+        case 41: 
+          { positionInc = 1; return EMAIL;
           }
         case 84: break;
-        case 36: 
-          { currentTokType = BOLD_ITALICS;  yybegin(FIVE_SINGLE_QUOTES_STATE);
+        case 28: 
+          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 85: break;
-        case 33: 
-          { positionInc = 1; return HOST;
+        case 23: 
+          { numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);/* Break so we don't hit fall-through warning: */ break;
           }
         case 86: break;
-        case 31: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
+        case 34: 
+          { positionInc = 1; return HOST;
           }
         case 87: break;
-        case 41: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
+        case 32: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;
           }
         case 88: break;
         case 12: 
           { currentTokType = ITALICS; numWikiTokensSeen++;  yybegin(STRING); return currentTokType;/*italics*/
           }
         case 89: break;
-        case 10: 
-          { numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
+        case 24: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);/* Break so we don't hit fall-through warning: */ break;
           }
         case 90: break;
+        case 31: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end italics*/
+          }
+        case 91: break;
+        case 26: 
+          { yybegin(YYINITIAL);/* Break so we don't hit fall-through warning: */ break;
+          }
+        case 92: break;
         default: 
           if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
             zzAtEOF = true;



Mime
View raw message