lucene-pylucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject svn commit: r1427274 - in /lucene/pylucene/trunk: Makefile test/PyLuceneTestCase.py test/test_Sort.py
Date Tue, 01 Jan 2013 01:30:56 GMT
Author: vajda
Date: Tue Jan  1 01:30:56 2013
New Revision: 1427274

URL: http://svn.apache.org/viewvc?rev=1427274&view=rev
Log:
migrated test/test_Sort.py

Modified:
    lucene/pylucene/trunk/Makefile
    lucene/pylucene/trunk/test/PyLuceneTestCase.py
    lucene/pylucene/trunk/test/test_Sort.py

Modified: lucene/pylucene/trunk/Makefile
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/Makefile?rev=1427274&r1=1427273&r2=1427274&view=diff
==============================================================================
--- lucene/pylucene/trunk/Makefile (original)
+++ lucene/pylucene/trunk/Makefile Tue Jan  1 01:30:56 2013
@@ -301,6 +301,7 @@ GENERATE=$(JCC) $(foreach jar,$(JARS),--
                      java.text.SimpleDateFormat \
                      java.text.DecimalFormat \
                      java.text.Collator \
+           --package java.util.concurrent java.util.concurrent.Executors \
            --package java.util.regex \
            --package java.io java.io.StringReader \
                              java.io.InputStreamReader \

Modified: lucene/pylucene/trunk/test/PyLuceneTestCase.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/PyLuceneTestCase.py?rev=1427274&r1=1427273&r2=1427274&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/PyLuceneTestCase.py (original)
+++ lucene/pylucene/trunk/test/PyLuceneTestCase.py Tue Jan  1 01:30:56 2013
@@ -43,7 +43,7 @@ class PyLuceneTestCase(TestCase):
         return IndexWriterConfig(self.TEST_VERSION, analyzer)
         
     def getWriter(self, directory=None, analyzer=None, open_mode=None,
-                  similarity=None):
+                  similarity=None, maxBufferedDocs=None, mergePolicy=None):
         if analyzer is None:
             analyzer = LimitTokenCountAnalyzer(WhitespaceAnalyzer(self.TEST_VERSION), 10000)
         config = self.getConfig(analyzer)
@@ -53,6 +53,10 @@ class PyLuceneTestCase(TestCase):
         config.setOpenMode(open_mode)
         if similarity is not None:
             config.setSimilarity(similarity)
+        if maxBufferedDocs is not None:
+            config.setMaxBufferedDocs(maxBufferedDocs)
+        if mergePolicy is not None:
+            config.setMergePolicy(mergePolicy)
 
         if directory is None:
             directory = self.directory

Modified: lucene/pylucene/trunk/test/test_Sort.py
URL: http://svn.apache.org/viewvc/lucene/pylucene/trunk/test/test_Sort.py?rev=1427274&r1=1427273&r2=1427274&view=diff
==============================================================================
--- lucene/pylucene/trunk/test/test_Sort.py (original)
+++ lucene/pylucene/trunk/test/test_Sort.py Tue Jan  1 01:30:56 2013
@@ -12,18 +12,38 @@
 #   limitations under the License.
 # ====================================================================
 
-import math
+import sys, lucene, unittest, math
+from PyLuceneTestCase import PyLuceneTestCase
 
 from itertools import izip
 from random import randint
-from unittest import TestCase, main
-from lucene import *
 
-NUM_STRINGS = 6000
+from java.lang import Byte, Double, Float, Integer, Long, Short
+from java.util import BitSet
+from java.util.concurrent import Executors, TimeUnit
+from org.apache.lucene.analysis.core import SimpleAnalyzer
+from org.apache.lucene.codecs import Codec
+from org.apache.lucene.document import \
+    Document, Field, FieldType, StringField, StoredField, TextField, \
+    PackedLongDocValuesField, FloatDocValuesField, SortedBytesDocValuesField, \
+    StraightBytesDocValuesField, DerefBytesDocValuesField, DoubleDocValuesField
+from org.apache.lucene.index import \
+    DocValues, LogDocMergePolicy, MultiReader, Term
+from org.apache.lucene.search import \
+    BooleanQuery, BooleanClause, FieldCache, IndexSearcher, MatchAllDocsQuery, Sort, \
+    SortField, TermQuery, TopFieldCollector
+from org.apache.lucene.store import RAMDirectory
+from org.apache.lucene.util import \
+    BytesRef, DocIdBitSet, FieldCacheSanityChecker, NamedThreadFactory, Version
+from org.apache.pylucene.search import \
+    PythonIntParser, PythonFloatParser, PythonLongParser, PythonDoubleParser, \
+    PythonByteParser, PythonShortParser, \
+    PythonFieldComparator, PythonFieldComparatorSource, PythonFilter
 
+NUM_STRINGS = 750
 
 
-class SortTestCase(TestCase):
+class SortTestCase(PyLuceneTestCase):
     """
     Unit tests for sorting code, ported from Java Lucene
     """
@@ -31,111 +51,200 @@ class SortTestCase(TestCase):
     def __init__(self, *args, **kwds):
 
         super(SortTestCase, self).__init__(*args, **kwds)
+        self.supportsDocValues = Codec.getDefault().getName() > "Lucene3x"
 
         self.data = [
-    #      tracer  contents         int            float           string   custom   i18n               long                  double,                short,                byte,           custom parser encoding'
-        [   "A",   "x a",           "5",           "4f",           "c",    "A-3",   u"p\u00EAche",      "10",                  "-4.0",                "3",                  "126",          "J"  ],
-        [   "B",   "y a",           "5",           "3.4028235E38", "i",    "B-10",  "HAT",             "1000000000",          "40.0",                "24",                 "1",            "I"  ],
-        [   "C",   "x a b c",       "2147483647",  "1.0",          "j",    "A-2",   u"p\u00E9ch\u00E9", "99999999",            "40.00002343",         "125",                "15",           "H"  ],
-        [   "D",   "y a b c",       "-1",          "0.0f",         "a",     "C-0",   "HUT",             str(Long.MAX_VALUE),  str(Double.MIN_VALUE), str(Short.MIN_VALUE), str(Byte.MIN_VALUE), "G"  ],
-        [   "E",   "x a b c d",     "5",           "2f",           "h",     "B-8",   "peach",           str(Long.MIN_VALUE),  str(Double.MAX_VALUE), str(Short.MAX_VALUE), str(Byte.MAX_VALUE), "F"  ],
-        [   "F",   "y a b c d",     "2",           "3.14159f",     "g",     "B-1",   u"H\u00C5T",        "-44",                "343.034435444",       "-3",                 "0",            "E"  ],
-        [   "G",   "x a b c d",     "3",           "-1.0",         "f",     "C-100", "sin",             "323254543543",       "4.043544",            "5",                  "100",          "D"  ],
-        [   "H",   "y a b c d",     "0",           "1.4E-45",      "e",     "C-88",  u"H\u00D8T",        "1023423423005",      "4.043545",            "10",                 "-50",          "C"  ],
-        [   "I",   "x a b c d e f", "-2147483648", "1.0e+0",       "d",     "A-10",  u"s\u00EDn",        "332422459999",       "4.043546",            "-340",               "51",           "B"  ],
-        [   "J",   "y a b c d e f", "4",           ".5",           "b",     "C-7",   "HOT",             "34334543543",        "4.0000220343",        "300",                "2",            "A"  ],
-        [   "W",   "g",             "1",           None,           None,    None,    None,              None,                 None,                  None,                 None,           None  ],
-        [   "X",   "g",             "1",           "0.1",          None,    None,    None,              None,                 None,                  None,                 None,           None  ],
-        [   "Y",   "g",             "1",           "0.2",          None,    None,    None,              None,                 None,                  None,                 None,           None  ],
-        [   "Z",   "f g",           None,          None,           None,    None,    None,              None,                 None,                  None,                 None,           None  ],
+            # tracer  contents         int            float           string   custom   i18n               long            double,          short,     byte, 'custom parser encoding'
+            [ "A",   "x a",           "5",           "4f",           "c",     "A-3",   "p\u00EAche",      "10",           "-4.0",            "3",    "126", "J" ],  # A, x
+            [ "B",   "y a",           "5",           "3.4028235E38", "i",     "B-10",  "HAT",             "1000000000",   "40.0",           "24",      "1", "I" ],  # B, y
+            [ "C",   "x a b c",       "2147483647",  "1.0",          "j",     "A-2",   "p\u00E9ch\u00E9", "99999999","40.00002343",        "125",     "15", "H" ],  # C, x
+            [ "D",   "y a b c",       "-1",          "0.0f",         "a",     "C-0",   "HUT",   str(Long.MAX_VALUE), str(Double.MIN_VALUE), str(Short.MIN_VALUE), str(Byte.MIN_VALUE), "G" ], # D, y
+            [ "E",   "x a b c d",     "5",           "2f",           "h",     "B-8",   "peach", str(Long.MIN_VALUE), str(Double.MAX_VALUE), str(Short.MAX_VALUE), str(Byte.MAX_VALUE), "F" ], # E, x
+            [ "F",   "y a b c d",     "2",           "3.14159f",     "g",     "B-1",   "H\u00C5T",        "-44",          "343.034435444",  "-3",      "0", "E" ],  # F, y
+            [ "G",   "x a b c d",     "3",           "-1.0",         "f",     "C-100", "sin",             "323254543543", "4.043544",        "5",    "100", "D" ],  # G, x
+            [ "H",   "y a b c d",     "0",           "1.4E-45",      "e",     "C-88",  "H\u00D8T",        "1023423423005","4.043545",       "10",    "-50", "C" ],  # H, y
+            [ "I",   "x a b c d e f", "-2147483648", "1.0e+0",       "d",     "A-10",  "s\u00EDn",        "332422459999", "4.043546",     "-340",     "51", "B" ],  # I, x
+            [ "J",   "y a b c d e f", "4",           ".5",           "b",     "C-7",   "HOT",             "34334543543",  "4.0000220343",  "300",      "2", "A" ],  # J, y
+            [ "W",   "g",             "1",           None,           None,    None,    None,              None,           None,             None,     None, None ],
+            [ "X",   "g",             "1",           "0.1",          None,    None,    None,              None,           None,             None,     None, None ],
+            [ "Y",   "g",             "1",           "0.2",          None,    None,    None,              None,           None,             None,     None, None ],
+            [ "Z",   "f g",           None,          None,           None,    None,    None,              None,           None,             None,     None, None ],
+  
+            # Sort Missing first/last
+            [ "a",   "m",             None,          None,           None,    None,    None,              None,           None,             None,     None, None ],
+            [ "b",   "m",            "4",           "4.0",           "4",     None,    None,              "4",            "4",               "4",      "4", None ],
+            [ "c",   "m",            "5",           "5.0",           "5",     None,    None,              "5",            "5",               "5",      "5", None ],
+            [ "d",   "m",            None,           None,           None,    None,    None,              None,           None,             None,     None, None ],
         ]
 
+    def setUp(self):
+        super(SortTestCase, self).setUp()
+
+        self.dirs = []
+        self.dvStringSorted = self.getRandomBoolean()
+        self.full = self._getFullIndex()
+        self.searchX = self._getXIndex()
+        self.searchY = self._getYIndex()
+        self.queryX = TermQuery(Term("contents", "x"))
+        self.queryY = TermQuery(Term("contents", "y"))
+        self.queryA = TermQuery(Term("contents", "a"))
+        self.queryE = TermQuery(Term("contents", "e"))
+        self.queryF = TermQuery(Term("contents", "f"))
+        self.queryG = TermQuery(Term("contents", "g"))
+        self.queryM = TermQuery(Term("contents", "m"))
+
+    def tearDown(self):
+
+        for directory in self.dirs:
+            directory.close()
+
+        super(SortTestCase, self).tearDown()
+
     def _getIndex(self, even, odd):
 
-        indexStore = RAMDirectory()
-        writer = IndexWriter(indexStore, SimpleAnalyzer(), True,
-                             IndexWriter.MaxFieldLength.LIMITED)
-        writer.setMaxBufferedDocs(2)
-        writer.setMergeFactor(1000)
+        mergePolicy = LogDocMergePolicy()
+        mergePolicy.setMergeFactor(1000)
+        directory = RAMDirectory()
+        self.dirs.append(directory)
+
+        writer = self.getWriter(directory=directory,
+                                analyzer=SimpleAnalyzer(Version.LUCENE_CURRENT),
+                                maxBufferedDocs=2, mergePolicy=mergePolicy)
+
+        if self.dvStringSorted:
+            # Index sorted
+            stringDVType = (DocValues.Type.BYTES_FIXED_SORTED, DocValues.Type.BYTES_VAR_SORTED)[self.getRandomNumber(0, 1)]
+        else:
+            # Index non-sorted
+            if self.getRandomBoolean(): # Fixed
+                stringDVType = (DocValues.Type.BYTES_FIXED_DEREF, DocValues.Type.BYTES_FIXED_STRAIGHT)[self.getRandomNumber(0, 1)]
+            else: # Var
+                stringDVType = (DocValues.Type.BYTES_VAR_DEREF, DocValues.Type.BYTES_VAR_STRAIGHT)[self.getRandomNumber(0, 1)]
+
+        ft1 = FieldType()
+        ft1.setStored(True)
+        ft2 = FieldType()
+        ft2.setIndexed(True)
 
         for i in xrange(len(self.data)):
             if (i % 2 == 0 and even) or (i % 2 == 1 and odd):
                 doc = Document()
-                doc.add(Field("tracer", self.data[i][0], Field.Store.YES,
-                              Field.Index.NO))
-                doc.add(Field("contents", self.data[i][1], Field.Store.NO,
-                              Field.Index.ANALYZED))
+                doc.add(Field("tracer", self.data[i][0], ft1))
+                doc.add(TextField("contents", self.data[i][1], Field.Store.NO))
                 if self.data[i][2] is not None:
-                    doc.add(Field("int", self.data[i][2], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("int", self.data[i][2], Field.Store.NO))
+                    if self.supportsDocValues:
+                        doc.add(PackedLongDocValuesField("int", Long.parseLong(self.data[i][2])))
                 if self.data[i][3] is not None:
-                    doc.add(Field("float", self.data[i][3], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("float", self.data[i][3], Field.Store.NO))
+                    if self.supportsDocValues:
+                        doc.add(FloatDocValuesField("float", Float.parseFloat(self.data[i][3])))
+
                 if self.data[i][4] is not None:
-                    doc.add(Field("string", self.data[i][4], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("string", self.data[i][4], Field.Store.NO))
+                    if self.supportsDocValues:
+                        if stringDVType == DocValues.Type.BYTES_FIXED_SORTED:
+                            doc.add(SortedBytesDocValuesField("string", BytesRef(self.data[i][4]), True))
+                        elif stringDVType == DocValues.Type.BYTES_VAR_SORTED:
+                            doc.add(SortedBytesDocValuesField("string", BytesRef(self.data[i][4]), False))
+                        elif stringDVType == DocValues.Type.BYTES_FIXED_STRAIGHT:
+                            doc.add(StraightBytesDocValuesField("string", BytesRef(self.data[i][4]), True))
+                        elif stringDVType == DocValues.Type.BYTES_VAR_STRAIGHT:
+                            doc.add(StraightBytesDocValuesField("string", BytesRef(self.data[i][4]), False))
+                        elif stringDVType == DocValues.Type.BYTES_FIXED_DEREF:
+                            doc.add(DerefBytesDocValuesField("string", BytesRef(self.data[i][4]), True))
+                        elif stringDVType == DocValues.Type.BYTES_VAR_DEREF:
+                            doc.add(DerefBytesDocValuesField("string", BytesRef(self.data[i][4]), False))
+                        else:
+                            raise ValueError("unknown type " + stringDVType)
+
                 if self.data[i][5] is not None:
-                    doc.add(Field("custom", self.data[i][5], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("custom", self.data[i][5], Field.Store.NO))
                 if self.data[i][6] is not None:
-                    doc.add(Field("i18n", self.data[i][6], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("i18n", self.data[i][6], Field.Store.NO))
                 if self.data[i][7] is not None:
-                    doc.add(Field("long", self.data[i][7], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("long", self.data[i][7], Field.Store.NO))
                 if self.data[i][8] is not None:
-                    doc.add(Field("double", self.data[i][8], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("double", self.data[i][8], Field.Store.NO))
+                    if self.supportsDocValues:
+                        doc.add(DoubleDocValuesField("double", Double.parseDouble(self.data[i][8])))
                 if self.data[i][9] is not None:
-                    doc.add(Field("short", self.data[i][9], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("short", self.data[i][9], Field.Store.NO))
                 if self.data[i][10] is not None:
-                    doc.add(Field("byte", self.data[i][10], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
+                    doc.add(StringField("byte", self.data[i][10], Field.Store.NO))
                 if self.data[i][11] is not None:
-                    doc.add(Field("parser", self.data[i][11], Field.Store.NO,
-                                  Field.Index.NOT_ANALYZED))
-                doc.setBoost(2.0)  # produce some scores above 1.0
+                    doc.add(StringField("parser", self.data[i][11], Field.Store.NO))
+
+                for f in doc.getFields():
+                    if f.fieldType().indexed() and not f.fieldType().omitNorms():
+                        Field.cast_(f).setBoost(2.0)
+
                 writer.addDocument(doc)
-        # writer.commit()
+
+        reader = writer.getReader()
         writer.close()
-        s = IndexSearcher(indexStore, True)
-        s.setDefaultFieldSortScoring(True, True)
 
-        return s
+        return self.getSearcher(reader=reader)
 
     def _getFullIndex(self):
         return self._getIndex(True, True)
 
-    def getFullStrings(self):
+    def _getFullStrings(self):
 
-        indexStore = RAMDirectory()
-        writer = IndexWriter(indexStore, SimpleAnalyzer(), True,
-                             IndexWriter.MaxFieldLength.LIMITED)
-        writer.setMaxBufferedDocs(4)
-        writer.setMergeFactor(97)
+        mergePolicy = LogDocMergePolicy()
+        mergePolicy.setMergeFactor(97)
+        directory = RAMDirectory()
+        self.dirs.append(directory)
+
+        writer = self.getWriter(directory=directory,
+                                analyzer=SimpleAnalyzer(Version.LUCENE_CURRENT),
+                                maxBufferedDocs=4, mergePolicy=mergePolicy)
         
+        onlyStored = FieldType()
+        onlyStored.setStored(True)
+        fixedLen = self.getRandomNumber(2, 8)
+        fixedLen2 = self.getRandomNumber(1, 4)
+
         for i in xrange(NUM_STRINGS):
             doc = Document()
+
             num = self.getRandomCharString(self.getRandomNumber(2, 8), 48, 52)
-            doc.add(Field("tracer", num, Field.Store.YES, Field.Index.NO))
-            # doc.add(Field("contents", str(i), Field.Store.NO,
-            #         Field.Index.ANALYZED))
-            doc.add(Field("string", num, Field.Store.NO,
-                          Field.Index.NOT_ANALYZED))
+            doc.add(Field("tracer", num, onlyStored))
+            doc.add(StringField("string", num, Field.Store.NO))
+            if self.supportsDocValues:
+                doc.add(SortedBytesDocValuesField("string", BytesRef(num)))
+
             num2 = self.getRandomCharString(self.getRandomNumber(1, 4), 48, 50)
-            doc.add(Field("string2", num2, Field.Store.NO,
-                          Field.Index.NOT_ANALYZED))
-            doc.add(Field("tracer2", num2, Field.Store.YES, Field.Index.NO))
-            doc.setBoost(2.0)  # produce some scores above 1.0
-            writer.setMaxBufferedDocs(self.getRandomNumber(2, 12))
+            doc.add(StringField("string2", num2, Field.Store.NO))
+            if self.supportsDocValues:
+                doc.add(SortedBytesDocValuesField("string2", BytesRef(num2)))
+            doc.add(Field("tracer2", num2, onlyStored))
+            for f2 in doc.getFields():
+                if f2.fieldType().indexed() and not f2.fieldType().omitNorms():
+                    Field.cast_(f2).setBoost(2.0)
+
+            numFixed = self.getRandomCharString(fixedLen, 48, 52)
+            doc.add(Field("fixed_tracer", numFixed, onlyStored))
+            doc.add(StringField("string_fixed", numFixed, Field.Store.NO))
+            if self.supportsDocValues:
+                doc.add(SortedBytesDocValuesField("string_fixed", BytesRef(numFixed),
+                                                  True))
+
+            num2Fixed = self.getRandomCharString(fixedLen2, 48, 52)
+            doc.add(StringField("string2_fixed", num2Fixed, Field.Store.NO))
+            if self.supportsDocValues:
+                doc.add(SortedBytesDocValuesField("string2_fixed", BytesRef(num2Fixed),
+                                                  True))
+            doc.add(Field("tracer2_fixed", num2Fixed, onlyStored))
+            for f2 in doc.getFields():
+                if f2.fieldType().indexed() and not f2.fieldType().omitNorms():
+                    Field.cast_(f2).setBoost(2.0)
+
             writer.addDocument(doc)
-      
-        # writer.commit()
-        # print writer.getSegmentCount()
+
         writer.close()
 
-        return IndexSearcher(indexStore, True)
+        return self.getSearcher(directory=directory)
   
     def getRandomNumberString(self, num, low, high):
 
@@ -154,6 +263,10 @@ class SortTestCase(TestCase):
   
         return randint(low, high)
 
+    def getRandomBoolean(self):
+  
+        return randint(0, 1) == 1
+
     def _getXIndex(self):
         return self._getIndex(True, False)
 
@@ -163,18 +276,6 @@ class SortTestCase(TestCase):
     def _getEmptyIndex(self):
         return self._getIndex(False, False)
 
-    def setUp(self):
-
-        self.full = self._getFullIndex()
-        self.searchX = self._getXIndex()
-        self.searchY = self._getYIndex()
-        self.queryX = TermQuery(Term("contents", "x"))
-        self.queryY = TermQuery(Term("contents", "y"))
-        self.queryA = TermQuery(Term("contents", "a"))
-        self.queryE = TermQuery(Term("contents", "e"))
-        self.queryF = TermQuery(Term("contents", "f"))
-        self.queryG = TermQuery(Term("contents", "g"))
-
     def testBuiltInSorts(self):
         """
         test the sorts by score and document number
@@ -192,96 +293,141 @@ class SortTestCase(TestCase):
         """
         test sorts where the type of field is specified
         """
-
         sort = Sort()
 
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField.FIELD_DOC])
+        sort.setSort([SortField("int", SortField.Type.INT), SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "IGAEC")
         self._assertMatches(self.full, self.queryY, sort, "DHFJB")
 
-        sort.setSort([SortField("float", SortField.FLOAT),
+        sort.setSort([SortField("float", SortField.Type.FLOAT),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "GCIEA")
         self._assertMatches(self.full, self.queryY, sort, "DHJFB")
 
-        sort.setSort([SortField("long", SortField.LONG),
+        sort.setSort([SortField("long", SortField.Type.LONG),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "EACGI")
         self._assertMatches(self.full, self.queryY, sort, "FBJHD")
 
-        sort.setSort([SortField("double", SortField.DOUBLE),
+        sort.setSort([SortField("double", SortField.Type.DOUBLE),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "AGICE")
         self._assertMatches(self.full, self.queryY, sort, "DJHBF")
 
-        sort.setSort([SortField("byte", SortField.BYTE),
+        sort.setSort([SortField("byte", SortField.Type.BYTE),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "CIGAE")
         self._assertMatches(self.full, self.queryY, sort, "DHFBJ")
 
-        sort.setSort([SortField("short", SortField.SHORT),
+        sort.setSort([SortField("short", SortField.Type.SHORT),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "IAGCE")
         self._assertMatches(self.full, self.queryY, sort, "DFHBJ")
 
-        sort.setSort([SortField("string", SortField.STRING),
+        sort.setSort([SortField("string", SortField.Type.STRING),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "AIGEC")
         self._assertMatches(self.full, self.queryY, sort, "DJHFB")
   
-    def testStringSort(self):
-        """
-        Test String sorting: small queue to many matches, multi field sort,
-        reverse sort
-        """
+    def _useDocValues(self, field):
 
-        sort = Sort()
-        searcher = self.getFullStrings()
+        field.setUseIndexValues(True)
+        return field
 
-        sort.setSort([SortField("string", SortField.STRING),
-                      SortField("string2", SortField.STRING, True),
-                      SortField.FIELD_DOC])
+    def _getDVStringSortType(self):
+
+        if self.dvStringSorted:
+            # If you index as sorted source you can still sort by value instead:
+            return (SortField.Type.STRING_VAL, SortField.Type.STRING_VAL)[self.getRandomNumber(0, 1)]
+        else:
+            return SortField.Type.STRING_VAL
 
-        result = searcher.search(MatchAllDocsQuery(), None, 500, sort).scoreDocs
+    def _verifyStringSort(self, sort):
+
+        searcher = self._getFullStrings()
+        result = searcher.search(MatchAllDocsQuery(), None,
+                                 self.getRandomNumber(500, searcher.getIndexReader().maxDoc()),
+                                 sort).scoreDocs
 
         buff = []
+        n = len(result)
         last = None
         lastSub = None
         lastDocId = 0
         fail = False
 
+        if sort.getSort()[0].getField().endswith("_fixed"):
+            fieldSuffix = "_fixed"
+        else:
+            fieldSuffix = ""
+
         for scoreDoc in result:
             doc2 = searcher.doc(scoreDoc.doc)
-            v = doc2.getValues("tracer")
-            v2 = doc2.getValues("tracer2")
+            v = doc2.getValues("tracer" + fieldSuffix)
+            v2 = doc2.getValues("tracer2" + fieldSuffix)
             for _v, _v2 in izip(v, v2):
+                buff.append(_v + "(" + _v2 + ")(" + str(scoreDoc.doc) + ")\n")
                 if last is not None:
                     _cmp = cmp(_v, last)
                     if _cmp < 0: # ensure first field is in order
                         fail = True
                         print "fail:", _v, "<", last
+                        buff.append("  WRONG tracer\n")
 
                     if _cmp == 0: # ensure second field is in reverse order
                         _cmp = cmp(_v2, lastSub)
                         if _cmp > 0:
                             fail = True
                             print "rev field fail:", _v2, ">", lastSub
+                            buff.append("  WRONG tracer2\n")
                         elif _cmp == 0: # ensure docid is in order
                             if scoreDoc.doc < lastDocId:
                                 fail = True
                                 print "doc fail:", scoreDoc.doc, ">", lastDocId
-
+                                buff.append("  WRONG docID\n")
                 last = _v
                 lastSub = _v2
                 lastDocId = scoreDoc.doc
-                buff.append(_v + "(" + _v2 + ")(" + str(scoreDoc.doc) + ") ")
 
         if fail:
             print "topn field1(field2)(docID):", ''.join(buff)
 
         self.assert_(not fail, "Found sort results out of order")
+        searcher.getIndexReader().close()
   
+    def testStringSort(self):
+        """
+        Test String sorting: small queue to many matches, multi field sort,
+        reverse sort
+        """
+
+        sort = Sort()
+
+        # Normal string field, var length
+        sort.setSort([SortField("string", SortField.Type.STRING),
+                      SortField("string2", SortField.Type.STRING, True),
+                      SortField.FIELD_DOC])
+        self._verifyStringSort(sort)
+
+        # Normal string field, fixed length
+        sort.setSort([SortField("string_fixed", SortField.Type.STRING),
+                      SortField("string2_fixed", SortField.Type.STRING, True),
+                      SortField.FIELD_DOC])
+        self._verifyStringSort(sort)
+
+        # Doc values field, var length
+        self.assertTrue(self.supportsDocValues, "cannot work with preflex codec")
+        sort.setSort([self._useDocValues(SortField("string", self._getDVStringSortType())),
+                      self._useDocValues(SortField("string2", self._getDVStringSortType(), True)),
+                      SortField.FIELD_DOC])
+        self._verifyStringSort(sort)
+
+        # Doc values field, fixed length
+        sort.setSort([self._useDocValues(SortField("string_fixed", self._getDVStringSortType())),
+                      self._useDocValues(SortField("string2_fixed", self._getDVStringSortType(), True)),
+                      SortField.FIELD_DOC])
+        self._verifyStringSort(sort)
+
     def testCustomFieldParserSort(self):
         """
         test sorts where the type of field is specified and a custom field
@@ -297,31 +443,33 @@ class SortTestCase(TestCase):
         
         class intParser(PythonIntParser):
             def parseInt(_self, val):
-                return (ord(val[0]) - ord('A')) * 123456
+                return (val.bytes[val.offset] - ord('A')) * 123456
 
         class floatParser(PythonFloatParser):
             def parseFloat(_self, val):
-                return math.sqrt(ord(val[0]))
+                return math.sqrt(val.bytes[val.offset])
 
         class longParser(PythonLongParser):
             def parseLong(_self, val):
-                return (ord(val[0]) - ord('A')) * 1234567890L
+                return (val.bytes[val.offset] - ord('A')) * 1234567890L
 
         class doubleParser(PythonDoubleParser):
             def parseDouble(_self, val):
-                return math.pow(ord(val[0]), ord(val[0]) - ord('A'))
+                return math.pow(val.bytes[val.offset], val.bytes[val.offset] - ord('A'))
 
         class byteParser(PythonByteParser):
             def parseByte(_self, val):
-                return chr(ord(val[0]) - ord('A'))
+                return chr(val.bytes[val.offset] - ord('A'))
 
         class shortParser(PythonShortParser):
             def parseShort(_self, val):
-                return ord(val[0]) - ord('A')
+                return val.bytes[val.offset] - ord('A')
 
         sort = Sort()
+
         sort.setSort([SortField("parser", intParser()),
                       SortField.FIELD_DOC])
+
         self._assertMatches(self.full, self.queryA, sort, "JIHGFEDCBA")
         self._assertSaneFieldCaches(self.getName() + " IntParser")
         fc.purgeAllCaches()
@@ -333,7 +481,7 @@ class SortTestCase(TestCase):
         fc.purgeAllCaches()
 
         sort.setSort([SortField("parser", longParser()),
-                           SortField.FIELD_DOC])
+                      SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryA, sort, "JIHGFEDCBA")
         self._assertSaneFieldCaches(self.getName() + " LongParser")
         fc.purgeAllCaches()
@@ -369,15 +517,15 @@ class SortTestCase(TestCase):
         sort.setSort(SortField.FIELD_DOC)
         self._assertMatches(empty, self.queryX, sort, "")
 
-        sort.setSort([SortField("int", SortField.INT), SortField.FIELD_DOC])
+        sort.setSort([SortField("int", SortField.Type.INT), SortField.FIELD_DOC])
         self._assertMatches(empty, self.queryX, sort, "")
 
-        sort.setSort([SortField("string", SortField.STRING, True),
+        sort.setSort([SortField("string", SortField.Type.STRING, True),
                       SortField.FIELD_DOC])
         self._assertMatches(empty, self.queryX, sort, "")
 
-        sort.setSort([SortField("float", SortField.FLOAT),
-                      SortField("string", SortField.STRING)])
+        sort.setSort([SortField("float", SortField.Type.FLOAT),
+                      SortField("string", SortField.Type.STRING)])
         self._assertMatches(empty, self.queryX, sort, "")
 
 
@@ -396,24 +544,24 @@ class SortTestCase(TestCase):
         """
         sort = Sort()
 
-        sort.setSort([SortField(None, SortField.SCORE, True),
+        sort.setSort([SortField(None, SortField.Type.SCORE, True),
                       SortField.FIELD_DOC])
         self._assertMatches(self.full, self.queryX, sort, "IEGCA")
         self._assertMatches(self.full, self.queryY, sort, "JFHDB")
 
-        sort.setSort(SortField(None, SortField.DOC, True))
+        sort.setSort(SortField(None, SortField.Type.DOC, True))
         self._assertMatches(self.full, self.queryX, sort, "IGECA")
         self._assertMatches(self.full, self.queryY, sort, "JHFDB")
 
-        sort.setSort(SortField("int", SortField.INT, True))
+        sort.setSort(SortField("int", SortField.Type.INT, True))
         self._assertMatches(self.full, self.queryX, sort, "CAEGI")
         self._assertMatches(self.full, self.queryY, sort, "BJFHD")
 
-        sort.setSort(SortField("float", SortField.FLOAT, True))
+        sort.setSort(SortField("float", SortField.Type.FLOAT, True))
         self._assertMatches(self.full, self.queryX, sort, "AECIG")
         self._assertMatches(self.full, self.queryY, sort, "BFJHD")
 
-        sort.setSort(SortField("string", SortField.STRING, True))
+        sort.setSort(SortField("string", SortField.Type.STRING, True))
         self._assertMatches(self.full, self.queryX, sort, "CEGIA")
         self._assertMatches(self.full, self.queryY, sort, "BFHJD")
 
@@ -424,80 +572,62 @@ class SortTestCase(TestCase):
         """
         sort = Sort()
 
-        sort.setSort(SortField("string", SortField.STRING))
+        sort.setSort(SortField("string", SortField.Type.STRING))
         self._assertMatches(self.full, self.queryF, sort, "ZJI")
 
-        sort.setSort(SortField("string", SortField.STRING, True))
+        sort.setSort(SortField("string", SortField.Type.STRING, True))
         self._assertMatches(self.full, self.queryF, sort, "IJZ")
     
-        sort.setSort(SortField("i18n", Locale.ENGLISH))
-        self._assertMatches(self.full, self.queryF, sort, "ZJI")
-    
-        sort.setSort(SortField("i18n", Locale.ENGLISH, True))
-        self._assertMatches(self.full, self.queryF, sort, "IJZ")
-
-        sort.setSort(SortField("int", SortField.INT))
+        sort.setSort(SortField("int", SortField.Type.INT))
         self._assertMatches(self.full, self.queryF, sort, "IZJ")
 
-        sort.setSort(SortField("int", SortField.INT, True))
+        sort.setSort(SortField("int", SortField.Type.INT, True))
         self._assertMatches(self.full, self.queryF, sort, "JZI")
 
-        sort.setSort(SortField("float", SortField.FLOAT))
+        sort.setSort(SortField("float", SortField.Type.FLOAT))
         self._assertMatches(self.full, self.queryF, sort, "ZJI")
 
         # using a nonexisting field as first sort key shouldn't make a
         # difference:
-        sort.setSort([SortField("nosuchfield", SortField.STRING),
-                      SortField("float", SortField.FLOAT)])
+        sort.setSort([SortField("nosuchfield", SortField.Type.STRING),
+                      SortField("float", SortField.Type.FLOAT)])
         self._assertMatches(self.full, self.queryF, sort, "ZJI")
 
-        sort.setSort(SortField("float", SortField.FLOAT, True))
+        sort.setSort(SortField("float", SortField.Type.FLOAT, True))
         self._assertMatches(self.full, self.queryF, sort, "IJZ")
 
         # When a field is None for both documents, the next SortField should
         # be used. 
         # Works for
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("string", SortField.STRING),
-                      SortField("float", SortField.FLOAT)])
+        sort.setSort([SortField("int", SortField.Type.INT),
+                      SortField("string", SortField.Type.STRING),
+                      SortField("float", SortField.Type.FLOAT)])
         self._assertMatches(self.full, self.queryG, sort, "ZWXY")
 
         # Reverse the last criterium to make sure the test didn't pass by
         # chance 
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("string", SortField.STRING),
-                      SortField("float", SortField.FLOAT, True)])
+        sort.setSort([SortField("int", SortField.Type.INT),
+                      SortField("string", SortField.Type.STRING),
+                      SortField("float", SortField.Type.FLOAT, True)])
         self._assertMatches(self.full, self.queryG, sort, "ZYXW")
 
-        # Do the same for a MultiSearcher
-        multiSearcher = MultiSearcher([self.full])
-
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("string", SortField.STRING),
-                      SortField("float", SortField.FLOAT)])
-        self._assertMatches(multiSearcher, self.queryG, sort, "ZWXY")
-
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("string", SortField.STRING),
-                      SortField("float", SortField.FLOAT, True)])
-        self._assertMatches(multiSearcher, self.queryG, sort, "ZYXW")
-
-        # Don't close the multiSearcher. it would close the full searcher too!
         # Do the same for a ParallelMultiSearcher
-        parallelSearcher = ParallelMultiSearcher([self.full])
 
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("string", SortField.STRING),
-                      SortField("float", SortField.FLOAT)])
+        threadPool = Executors.newFixedThreadPool(self.getRandomNumber(2, 8), NamedThreadFactory("testEmptyFieldSort"))
+        parallelSearcher=IndexSearcher(self.full.getIndexReader(), threadPool)
+
+        sort.setSort([SortField("int", SortField.Type.INT),
+                      SortField("string", SortField.Type.STRING),
+                      SortField("float", SortField.Type.FLOAT)])
         self._assertMatches(parallelSearcher, self.queryG, sort, "ZWXY")
 
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("string", SortField.STRING),
-                      SortField("float", SortField.FLOAT, True)])
+        sort.setSort([SortField("int", SortField.Type.INT),
+                      SortField("string", SortField.Type.STRING),
+                      SortField("float", SortField.Type.FLOAT, True)])
         self._assertMatches(parallelSearcher, self.queryG, sort, "ZYXW")
 
-        # Don't close the parallelSearcher. it would close the full searcher
-        # too!
+        threadPool.shutdown()
+        threadPool.awaitTermination(1000L, TimeUnit.MILLISECONDS)
 
     def testSortCombos(self):
         """
@@ -505,172 +635,30 @@ class SortTestCase(TestCase):
         """
         sort = Sort()
 
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("float", SortField.FLOAT)])
+        sort.setSort([SortField("int", SortField.Type.INT),
+                      SortField("float", SortField.Type.FLOAT)])
         self._assertMatches(self.full, self.queryX, sort, "IGEAC")
 
-        sort.setSort([SortField("int", SortField.INT, True),
-                      SortField(None, SortField.DOC, True)])
+        sort.setSort([SortField("int", SortField.Type.INT, True),
+                      SortField(None, SortField.Type.DOC, True)])
         self._assertMatches(self.full, self.queryX, sort, "CEAGI")
 
-        sort.setSort([SortField("float", SortField.FLOAT),
-                      SortField("string", SortField.STRING)])
+        sort.setSort([SortField("float", SortField.Type.FLOAT),
+                      SortField("string", SortField.Type.STRING)])
         self._assertMatches(self.full, self.queryX, sort, "GICEA")
 
-    def testLocaleSort(self):
-        """
-        test using a Locale for sorting strings
-        """
-        sort = Sort()
-
-        sort.setSort([SortField("string", Locale.US)])
-        self._assertMatches(self.full, self.queryX, sort, "AIGEC")
-        self._assertMatches(self.full, self.queryY, sort, "DJHFB")
-
-        sort.setSort([SortField("string", Locale.US, True)])
-        self._assertMatches(self.full, self.queryX, sort, "CEGIA")
-        self._assertMatches(self.full, self.queryY, sort, "BFHJD")
-
-    def testInternationalSort(self):
-        """
-        test using various international locales with accented characters
-        (which sort differently depending on locale)
-        """
-        sort = Sort()
-
-        sort.setSort(SortField("i18n", Locale.US))
-        self._assertMatches(self.full, self.queryY, sort, "BFJDH")
-
-        sort.setSort(SortField("i18n", Locale("sv", "se")))
-        self._assertMatches(self.full, self.queryY, sort, "BJDFH")
-
-        sort.setSort(SortField("i18n", Locale("da", "dk")))
-        self._assertMatches(self.full, self.queryY, sort, "BJDHF")
-
-        sort.setSort(SortField("i18n", Locale.US))
-        self._assertMatches(self.full, self.queryX, sort, "ECAGI")
-
-        sort.setSort(SortField("i18n", Locale.FRANCE))
-        self._assertMatches(self.full, self.queryX, sort, "EACGI")
-
-    def testInternationalMultiSearcherSort(self):
-        """
-        Test the MultiSearcher's ability to preserve locale-sensitive ordering
-        by wrapping it around a single searcher
-        """
-        sort = Sort()
-
-        multiSearcher = MultiSearcher([self.full])
-        sort.setSort(SortField("i18n", Locale("sv", "se")))
-        self._assertMatches(multiSearcher, self.queryY, sort, "BJDFH")
-    
-        sort.setSort(SortField("i18n", Locale.US))
-        self._assertMatches(multiSearcher, self.queryY, sort, "BFJDH")
-    
-        sort.setSort(SortField("i18n", Locale("da", "dk")))
-        self._assertMatches(multiSearcher, self.queryY, sort, "BJDHF")
-    
-    def testMultiSort(self):
-        """
-        test a variety of sorts using more than one searcher
-        """
-        
-        searcher = MultiSearcher([self.searchX, self.searchY])
-        self.runMultiSorts(searcher, False)
-
     def testParallelMultiSort(self):
         """
         test a variety of sorts using a parallel multisearcher
         """
+        threadPool = Executors.newFixedThreadPool(self.getRandomNumber(2, 8), NamedThreadFactory("testParallelMultiSort"))
+        searcher = IndexSearcher(MultiReader([self.searchX.getIndexReader(),
+                                              self.searchY.getIndexReader()]),
+                                 threadPool)
+        self._runMultiSorts(searcher, False)
 
-        searcher = ParallelMultiSearcher([self.searchX, self.searchY])
-        self.runMultiSorts(searcher, False)
-
-    def testNormalizedScores(self):
-        """
-        test that the relevancy scores are the same even if
-        hits are sorted
-        """
-
-        # capture relevancy scores
-        scoresX = self.getScores(self.full.search(self.queryX, None,
-                                                  1000).scoreDocs, self.full)
-        scoresY = self.getScores(self.full.search(self.queryY, None,
-                                                  1000).scoreDocs, self.full)
-        scoresA = self.getScores(self.full.search(self.queryA, None,
-                                                  1000).scoreDocs, self.full)
-
-        # we'll test searching locally, remote and multi
-        multi = MultiSearcher([self.searchX, self.searchY])
-
-        # change sorting and make sure relevancy stays the same
-
-        sort = Sort()
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort(SortField.FIELD_DOC)
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort(SortField("int", SortField.INT))
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort(SortField("float", SortField.FLOAT))
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort(SortField("string", SortField.STRING))
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("float", SortField.FLOAT)])
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort([SortField("int", SortField.INT, True),
-                      SortField(None, SortField.DOC, True)])
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
-
-        sort.setSort([SortField("float", SortField.FLOAT),
-                      SortField("string", SortField.STRING)])
-        self._assertSameValues(scoresX, self.getScores(self.full.search(self.queryX, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresX, self.getScores(multi.search(self.queryX, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresY, self.getScores(self.full.search(self.queryY, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresY, self.getScores(multi.search(self.queryY, None, 1000, sort).scoreDocs, multi))
-        self._assertSameValues(scoresA, self.getScores(self.full.search(self.queryA, None, 1000, sort).scoreDocs, self.full))
-        self._assertSameValues(scoresA, self.getScores(multi.search(self.queryA, None, 1000, sort).scoreDocs, multi))
+        threadPool.shutdown();
+        threadPool.awaitTermination(1000L, TimeUnit.MILLISECONDS);
 
     def testTopDocsScores(self):
         """
@@ -683,20 +671,19 @@ class SortTestCase(TestCase):
 
         # try to pick a query that will result in an unnormalized
         # score greater than 1 to test for correct normalization
-        docs1 = self.full.search(self.queryE, None, nDocs, sort)
+        docs1 = self.full.search(self.queryE, None, nDocs, sort, True, True)
 
         # a filter that only allows through the first hit
         class filter(PythonFilter):
-            def getDocIdSet(_self, context):
-                reader = context.reader
+            def getDocIdSet(_self, context, acceptDocs):
+                reader = context.reader()
                 bs = BitSet(reader.maxDoc())
                 bs.set(0, reader.maxDoc())
                 bs.set(docs1.scoreDocs[0].doc)
                 return DocIdBitSet(bs)
 
-        filt = filter()
+        docs2 = self.full.search(self.queryE, filter(), nDocs, sort, True, True)
 
-        docs2 = self.full.search(self.queryE, filt, nDocs, sort)
         self.assertEqual(docs1.scoreDocs[0].score,
                          docs2.scoreDocs[0].score,
                          1e-6)
@@ -845,7 +832,7 @@ class SortTestCase(TestCase):
             self.assertEqual(0, tds.totalHits)
             self.assert_(Float.isNaN_(tds.getMaxScore()))
   
-    def runMultiSorts(self, multi, isFull):
+    def _runMultiSorts(self, multi, isFull):
         """
         runs a variety of sorts useful for multisearchers
         """
@@ -855,78 +842,64 @@ class SortTestCase(TestCase):
         expected = isFull and "ABCDEFGHIJ" or "ACEGIBDFHJ"
         self._assertMatches(multi, self.queryA, sort, expected)
 
-        sort.setSort(SortField("int", SortField.INT))
+        sort.setSort(SortField("int", SortField.Type.INT))
         expected = isFull and "IDHFGJABEC" or "IDHFGJAEBC"
         self._assertMatches(multi, self.queryA, sort, expected)
 
-        sort.setSort([SortField("int", SortField.INT), SortField.FIELD_DOC])
+        sort.setSort([SortField("int", SortField.Type.INT), SortField.FIELD_DOC])
         expected = isFull and "IDHFGJABEC" or "IDHFGJAEBC"
         self._assertMatches(multi, self.queryA, sort, expected)
 
-        sort.setSort(SortField("int", SortField.INT))
+        sort.setSort(SortField("int", SortField.Type.INT))
         expected = isFull and "IDHFGJABEC" or "IDHFGJAEBC"
         self._assertMatches(multi, self.queryA, sort, expected)
 
-        sort.setSort([SortField("float", SortField.FLOAT), SortField.FIELD_DOC])
+        sort.setSort([SortField("float", SortField.Type.FLOAT), SortField.FIELD_DOC])
         self._assertMatches(multi, self.queryA, sort, "GDHJCIEFAB")
 
-        sort.setSort(SortField("float", SortField.FLOAT))
+        sort.setSort(SortField("float", SortField.Type.FLOAT))
         self._assertMatches(multi, self.queryA, sort, "GDHJCIEFAB")
 
-        sort.setSort(SortField("string", SortField.STRING))
+        sort.setSort(SortField("string", SortField.Type.STRING))
         self._assertMatches(multi, self.queryA, sort, "DJAIHGFEBC")
 
-        sort.setSort(SortField("int", SortField.INT, True))
+        sort.setSort(SortField("int", SortField.Type.INT, True))
         expected = isFull and "CABEJGFHDI" or "CAEBJGFHDI"
         self._assertMatches(multi, self.queryA, sort, expected)
 
-        sort.setSort(SortField("float", SortField.FLOAT, True))
+        sort.setSort(SortField("float", SortField.Type.FLOAT, True))
         self._assertMatches(multi, self.queryA, sort, "BAFECIJHDG")
 
-        sort.setSort(SortField("string", SortField.STRING, True))
+        sort.setSort(SortField("string", SortField.Type.STRING, True))
         self._assertMatches(multi, self.queryA, sort, "CBEFGHIAJD")
 
-        sort.setSort([SortField("int", SortField.INT),
-                      SortField("float", SortField.FLOAT)])
+        sort.setSort([SortField("int", SortField.Type.INT),
+                      SortField("float", SortField.Type.FLOAT)])
         self._assertMatches(multi, self.queryA, sort, "IDHFGJEABC")
 
-        sort.setSort([SortField("float", SortField.FLOAT),
-                      SortField("string", SortField.STRING)])
+        sort.setSort([SortField("float", SortField.Type.FLOAT),
+                      SortField("string", SortField.Type.STRING)])
         self._assertMatches(multi, self.queryA, sort, "GDHJICEFAB")
 
-        sort.setSort(SortField("int", SortField.INT))
+        sort.setSort(SortField("int", SortField.Type.INT))
         self._assertMatches(multi, self.queryF, sort, "IZJ")
 
-        sort.setSort(SortField("int", SortField.INT, True))
+        sort.setSort(SortField("int", SortField.Type.INT, True))
         self._assertMatches(multi, self.queryF, sort, "JZI")
 
-        sort.setSort(SortField("float", SortField.FLOAT))
+        sort.setSort(SortField("float", SortField.Type.FLOAT))
         self._assertMatches(multi, self.queryF, sort, "ZJI")
 
-        sort.setSort(SortField("string", SortField.STRING))
+        sort.setSort(SortField("string", SortField.Type.STRING))
         self._assertMatches(multi, self.queryF, sort, "ZJI")
 
-        sort.setSort(SortField("string", SortField.STRING, True))
+        sort.setSort(SortField("string", SortField.Type.STRING, True))
         self._assertMatches(multi, self.queryF, sort, "IJZ")
 
         # up to this point, all of the searches should have "sane" 
         # FieldCache behavior, and should have reused hte cache in several
         # cases 
         self._assertSaneFieldCaches(self.getName() + " various")
-        
-        # next we'll check Locale based(String[]) for 'string', so purge first
-        FieldCache.DEFAULT.purgeAllCaches()
-
-        sort.setSort([SortField("string", Locale.US)])
-        self._assertMatches(multi, self.queryA, sort, "DJAIHGFEBC")
-
-        sort.setSort([SortField("string", Locale.US, True)])
-        self._assertMatches(multi, self.queryA, sort, "CBEFGHIAJD")
-
-        sort.setSort([SortField("string", Locale.UK)])
-        self._assertMatches(multi, self.queryA, sort, "DJAIHGFEBC")
-
-        self._assertSaneFieldCaches(self.getName() + " Locale.US + Locale.UK")
         FieldCache.DEFAULT.purgeAllCaches()
 
     def _assertMatches(self, searcher, query, sort, expectedResult):
@@ -1004,20 +977,28 @@ class MyFieldComparator(PythonFieldCompa
         
         class intParser(PythonIntParser):
             def parseInt(_self, val):
-                return (ord(val[0]) - ord('A')) * 123456
+                return (val.bytes[val.offset] - ord('A')) * 123456
                 
-        self.docValues = FieldCache.DEFAULT.getInts(context.reader, "parser",
-                                                    intParser())
+        self.docValues = FieldCache.DEFAULT.getInts(context.reader(), "parser",
+                                                    intParser(), False)
 
         return self
 
     def value(self, slot):
         return Integer(self.slotValues[slot])
 
+    def compareDocToValue(self, doc, valueObj):
+        value = valueObj.intValue()
+        docValue = self.docValues[doc]
+
+        # values are small enough that overflow won't happen
+        return docValue - value
+
 
 class MyFieldComparatorSource(PythonFieldComparatorSource):
 
     def newComparator(self, fieldname, numHits, sortPos, reversed):
+
         # keep an extra ref since this object seems to be passed around
         # back and forth without a reference being kept on the java side
         self.saved = MyFieldComparator(numHits)
@@ -1025,17 +1006,16 @@ class MyFieldComparatorSource(PythonFiel
 
 
 if __name__ == "__main__":
-    import sys, lucene
     env = lucene.initVM()
     if '-loop' in sys.argv:
         sys.argv.remove('-loop')
         while True:
             try:
-                main()
+                unittest.main()
             except:
                 pass
 #            refs = sorted(env._dumpRefs(classes=True).items(),
 #                          key=lambda x: x[1], reverse=True)
 #            print refs[0:4]
     else:
-        main()
+        unittest.main()



Mime
View raw message